diff --git a/lib/internal/test_runner/harness.js b/lib/internal/test_runner/harness.js index 6b3b13b2c88d65..ecee5cdefaa897 100644 --- a/lib/internal/test_runner/harness.js +++ b/lib/internal/test_runner/harness.js @@ -61,6 +61,7 @@ function createTestTree(rootTestOptions, globalOptions) { failed: 0, passed: 0, cancelled: 0, + flaky: 0, skipped: 0, todo: 0, topLevel: 0, @@ -377,7 +378,7 @@ function runInParentContext(Factory) { return run(name, options, fn, overrides); }; - ArrayPrototypeForEach(['expectFailure', 'skip', 'todo', 'only'], (keyword) => { + ArrayPrototypeForEach(['expectFailure', 'flaky', 'skip', 'todo', 'only'], (keyword) => { test[keyword] = (name, options, fn) => { const overrides = { __proto__: null, diff --git a/lib/internal/test_runner/reporter/tap.js b/lib/internal/test_runner/reporter/tap.js index 01c698871b9134..595c96b48c8970 100644 --- a/lib/internal/test_runner/reporter/tap.js +++ b/lib/internal/test_runner/reporter/tap.js @@ -33,12 +33,12 @@ async function * tapReporter(source) { for await (const { type, data } of source) { switch (type) { case 'test:fail': { - yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure); + yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure, data.flaky); const location = data.file ? `${data.file}:${data.line}:${data.column}` : null; yield reportDetails(data.nesting, data.details, location); break; } case 'test:pass': - yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure); + yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure, data.flaky); yield reportDetails(data.nesting, data.details, null); break; case 'test:plan': @@ -65,7 +65,7 @@ async function * tapReporter(source) { } } -function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure) { +function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure, flaky) { let line = `${indent(nesting)}${status} ${testNumber}`; if (name) { @@ -78,6 +78,10 @@ function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`; } else if (expectFailure !== undefined) { line += ' # EXPECTED FAILURE'; + //should we use flaky >=0 here? for always printing 0 retries + } else if (flaky !== undefined && flaky > 0) { + const retryText = flaky === 1 ? 're-try' : 're-tries'; + line += ` # FLAKY ${flaky} ${retryText}`; } line += '\n'; diff --git a/lib/internal/test_runner/reporter/utils.js b/lib/internal/test_runner/reporter/utils.js index d90040b9727aa2..4f0f888572c2d1 100644 --- a/lib/internal/test_runner/reporter/utils.js +++ b/lib/internal/test_runner/reporter/utils.js @@ -71,7 +71,7 @@ function formatError(error, indent) { function formatTestReport(type, data, showErrorDetails = true, prefix = '', indent = '') { let color = reporterColorMap[type] ?? colors.white; let symbol = reporterUnicodeSymbolMap[type] ?? ' '; - const { skip, todo, expectFailure } = data; + const { skip, todo, expectFailure, flaky } = data; const duration_ms = data.details?.duration_ms ? ` ${colors.gray}(${data.details.duration_ms}ms)${colors.white}` : ''; let title = `${data.name}${duration_ms}`; @@ -87,6 +87,9 @@ function formatTestReport(type, data, showErrorDetails = true, prefix = '', inde } } else if (expectFailure !== undefined) { title += ` # EXPECTED FAILURE`; + } else if (flaky !== undefined && flaky > 0) { + const retryText = flaky === 1 ? 're-try' : 're-tries'; + title += ` # FLAKY ${flaky} ${retryText}`; } const err = showErrorDetails && data.details?.error ? formatError(data.details.error, indent) : ''; diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index beeb49c1763473..dffa7f0980d876 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -85,6 +85,7 @@ const kTestTimeoutFailure = 'testTimeoutFailure'; const kExpectedFailure = 'expectedFailure'; const kHookFailure = 'hookFailed'; const kDefaultTimeout = null; +const kDefaultFlakyRetries = 20; const noop = FunctionPrototype; const kShouldAbort = Symbol('kShouldAbort'); const kHookNames = ObjectSeal(['before', 'after', 'beforeEach', 'afterEach']); @@ -497,7 +498,8 @@ class Test extends AsyncResource { super('Test'); let { fn, name, parent } = options; - const { concurrency, entryFile, expectFailure, loc, only, timeout, todo, skip, signal, plan } = options; + + const { concurrency, entryFile, expectFailure, flaky, loc, only, timeout, todo, skip, signal, plan } = options; if (typeof fn !== 'function') { fn = noop; diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js index 5b53342933cdcb..30ded02c25c3de 100644 --- a/lib/internal/test_runner/utils.js +++ b/lib/internal/test_runner/utils.js @@ -401,6 +401,10 @@ function countCompletedTest(test, harness = test.root.harness) { } else { harness.counters.passed++; } + + if (test.flakyRetries > 0) { + harness.counters.flaky++; + } harness.counters.tests++; } diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000000000..93bc0662b7b834 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "node", + "lockfileVersion": 3, + "requires": true, + "packages": {} +}