test_runner: support bail out at runner level

lib/internal/test_runner/harness.js

@@ -52,6 +52,8 @@ function createTestTree(rootTestOptions, globalOptions) {
     buildSuites: [],
     isWaitingForBuildPhase: false,
     watching: false,
+    bail: globalOptions.bail,
+    bailedOut: false,
     config: globalOptions,
     coverage: null,
     resetCounters() {

lib/internal/test_runner/reporter/spec.js

@@ -78,6 +78,8 @@ class SpecReporter extends Transform {
   }
   #handleEvent({ type, data }) {
     switch (type) {
+      case 'test:bail':
+        return `${reporterColorMap['test:bail']}Bailing out! no new test files will be started!${colors.white}\n`;
       case 'test:fail':
         if (data.details?.error?.failureType !== kSubtestsFailed) {
           ArrayPrototypePush(this.#failedTests, data);

lib/internal/test_runner/reporter/utils.js

@@ -37,6 +37,9 @@ const reporterColorMap = {
   get 'test:diagnostic'() {
     return colors.blue;
   },
+  get 'test:bail'() {
+    return colors.yellow;
+  },
   get 'info'() {
     return colors.blue;
   },

lib/internal/test_runner/runner.js

@@ -22,6 +22,8 @@ const {
   SafePromiseAll,
   SafePromiseAllReturnVoid,
   SafePromiseAllSettledReturnVoid,
+  SafePromisePrototypeFinally,
+  SafePromiseRace,
   SafeSet,
   StringPrototypeIndexOf,
   StringPrototypeSlice,
@@ -254,6 +256,7 @@ class FileTest extends Test {
     if (item.data.details?.error) {
       item.data.details.error = deserializeError(item.data.details.error);
     }
+
     if (item.type === 'test:pass' || item.type === 'test:fail') {
       item.data.testNumber = isTopLevel ? (this.root.harness.counters.topLevel + 1) : item.data.testNumber;
       countCompletedTest({
@@ -604,6 +607,7 @@ function run(options = kEmptyObject) {
   } = options;
   const {
     concurrency,
+    bail,
     timeout,
     signal,
     files,
@@ -747,6 +751,7 @@ function run(options = kEmptyObject) {
     functionCoverage: functionCoverage,
     cwd,
     globalSetupPath,
+    bail,
   };
   const root = createTestTree(rootTestOptions, globalOptions);
   let testFiles = files ?? createTestFileList(globPatterns, cwd);
@@ -756,6 +761,16 @@ function run(options = kEmptyObject) {
     testFiles = ArrayPrototypeFilter(testFiles, (_, index) => index % shard.total === shard.index - 1);
   }
 
+  if (bail) {
+    validateBoolean(bail, 'options.bail');
+    if (watch) {
+      throw new ERR_INVALID_ARG_VALUE('options.bail', watch, 'bail not supported with watch mode');
+    }
+    if (isolation === 'none') {
+      throw new ERR_INVALID_ARG_VALUE('options.bail', isolation, 'bail not supported with \'none\' isolation');
+    }
+  }
+
   let teardown;
   let postRun;
   let filesWatcher;
@@ -770,6 +785,7 @@ function run(options = kEmptyObject) {
     hasFiles: files != null,
     globPatterns,
     only,
+    bail,
     forceExit,
     cwd,
     isolation,
@@ -792,15 +808,66 @@ function run(options = kEmptyObject) {
       teardown = () => root.harness.teardown();
     }
 
-    runFiles = () => {
-      root.harness.bootstrapPromise = null;
-      root.harness.buildPromise = null;
-      return SafePromiseAllSettledReturnVoid(testFiles, (path) => {
-        const subtest = runTestFile(path, filesWatcher, opts);
-        filesWatcher?.runningSubtests.set(path, subtest);
-        return subtest;
+    if (bail) {
+      runFiles = async () => {
+        root.harness.bootstrapPromise = null;
+        root.harness.buildPromise = null;
+
+        const running = new SafeSet();
+        let index = 0;
+
+        const shouldBail = () => bail && root.harness.bailedOut;
+
+        const enqueueNext = () => {
+          if (index < testFiles.length && !shouldBail()) {
+            const path = testFiles[index++];
+            const subtest = runTestFile(path, filesWatcher, opts);
+            filesWatcher?.runningSubtests.set(path, subtest);
+            running.add(subtest);
+            SafePromisePrototypeFinally(subtest, () => running.delete(subtest));
+          }
+        };
+
+        // Fill initial pool up to root test concurrency
+        // We use root test concurrency here because concurrency logic is handled at test level.
+        while (running.size < root.concurrency && index < testFiles.length && !shouldBail()) {
+          enqueueNext();
+        }
+
+        // As each test completes, enqueue the next one
+        while (running.size > 0) {
+          await SafePromiseRace([...running]);
+
+          // Refill pool after completion(s)
+          while (running.size < root.concurrency && index < testFiles.length && !shouldBail()) {
+            enqueueNext();
+          }
+        }
+      };
+
+      root.reporter.on('test:fail', (item) => {
+        if (root.harness.bail && !root.harness.bailedOut) {
+          process.nextTick(() => {
+            root.harness.bailedOut = true;
+            root.reporter[kEmitMessage]('test:bail', {
+              __proto__: null,
+              file: item.name,
+              test: item.data,
+            });
+          });
+        }
       });
-    };
+    } else {
+      runFiles = () => {
+        root.harness.bootstrapPromise = null;
+        root.harness.buildPromise = null;
+        return SafePromiseAllSettledReturnVoid(testFiles, (path) => {
+          const subtest = runTestFile(path, filesWatcher, opts);
+          filesWatcher?.runningSubtests.set(path, subtest);
+          return subtest;
+        });
+      };
+    }
   } else if (isolation === 'none') {
     if (watch) {
       const absoluteTestFiles = ArrayPrototypeMap(testFiles, (file) => (isAbsolute(file) ? file : resolve(cwd, file)));

lib/internal/test_runner/test.js

@@ -523,7 +523,7 @@ class Test extends AsyncResource {
       this.root = this;
       this.harness = options.harness;
       this.config = this.harness.config;
-      this.concurrency = 1;
+      this.concurrency = 1; // <-- is this needed?
       this.nesting = 0;
       this.only = this.config.only;
       this.reporter = new TestsStream();
@@ -540,7 +540,7 @@ class Test extends AsyncResource {
       this.root = parent.root;
       this.harness = null;
       this.config = config;
-      this.concurrency = parent.concurrency;
+      this.concurrency = parent.concurrency; // <-- is this needed?
       this.nesting = nesting;
       this.only = only;
       this.reporter = parent.reporter;
@@ -580,7 +580,7 @@ class Test extends AsyncResource {
       }
     }
 
-    switch (typeof concurrency) {
+    switch (typeof concurrency) { // <-- here we are overriding this.concurrency with the value from options!
       case 'number':
         validateUint32(concurrency, 'options.concurrency', true);
         this.concurrency = concurrency;

lib/internal/test_runner/tests_stream.js

@@ -149,6 +149,15 @@ class TestsStream extends Readable {
     });
   }
 
+  bail(nesting, loc, test) {
+    this[kEmitMessage]('test:bail', {
+      __proto__: null,
+      nesting,
+      test,
+      ...loc,
+    });
+  }
+
   end() {
     this.#tryPush(null);
   }

lib/internal/test_runner/utils.js

@@ -211,6 +211,7 @@ function parseCommandLine() {
   }
 
   const isTestRunner = getOptionValue('--test');
+  const bail = getOptionValue('--test-bail');
   const coverage = getOptionValue('--experimental-test-coverage');
   const forceExit = getOptionValue('--test-force-exit');
   const sourceMaps = getOptionValue('--enable-source-maps');
@@ -341,6 +342,7 @@ function parseCommandLine() {
   globalTestOptions = {
     __proto__: null,
     isTestRunner,
+    bail,
     concurrency,
     coverage,
     coverageExcludeGlobs,

src/node_options.cc

@@ -861,6 +861,12 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
             kDisallowedInEnvvar,
             false,
             OptionNamespaces::kTestRunnerNamespace);
+  AddOption("--test-bail",
+            "abort test execution after first failure",
+            &EnvironmentOptions::test_runner_bail,
+            kDisallowedInEnvvar,
+            false,
+            OptionNamespaces::kTestRunnerNamespace);
   AddOption("--test-concurrency",
             "specify test runner concurrency",
             &EnvironmentOptions::test_runner_concurrency,

src/node_options.h

@@ -190,6 +190,7 @@ class EnvironmentOptions : public Options {
   std::vector<std::string> optional_env_file;
   bool has_env_file_string = false;
   bool test_runner = false;
+  bool test_runner_bail = false;
   uint64_t test_runner_concurrency = 0;
   uint64_t test_runner_timeout = 0;
   bool test_runner_coverage = false;

test/fixtures/test-runner/bail/bail-test-1-pass.js

@@ -0,0 +1,12 @@
+'use strict';
+const { test } = require('node:test');
+const { setTimeout } = require('timers/promises');
+
+test('test 1 passes', async () => {
+  // This should pass
+  await setTimeout(500);
+});
+
+test('test 2 passes', () => {
+  // This should pass
+});

test/fixtures/test-runner/bail/bail-test-2-fail.js

@@ -0,0 +1,11 @@
+'use strict';
+const { test } = require('node:test');
+const assert = require('assert');
+
+test('failing test 1', () => {
+  assert.strictEqual(1, 2, 'This test should fail');
+});
+
+test('failing test 2', () => {
+  assert.strictEqual(3, 4, 'This test fails as well');
+});

test/fixtures/test-runner/bail/bail-test-3-pass.js

@@ -0,0 +1,10 @@
+'use strict';
+const { test } = require('node:test');
+
+test('test 3 passes', () => {
+  // This should not run if bail happens in test 2
+});
+
+test('test 4 passes', () => {
+  // This should not run if bail happens in test 2
+});

test/fixtures/test-runner/bail/bail-test-4-pass.js

@@ -0,0 +1,6 @@
+'use strict';
+const { test } = require('node:test');
+
+test('test 5 passes', () => {
+  // This should not run if bail happens earlier
+});

test/fixtures/test-runner/output/bail_concurrency_1.js

@@ -0,0 +1,13 @@
+'use strict';
+const fixtures = require('../../../common/fixtures');
+const { spec } = require('node:test/reporters');
+const { run } = require('node:test');
+
+const files = [
+  fixtures.path('test-runner', 'bail', 'bail-test-1-pass.js'),
+  fixtures.path('test-runner', 'bail', 'bail-test-2-fail.js'),
+  fixtures.path('test-runner', 'bail', 'bail-test-3-pass.js'),
+  fixtures.path('test-runner', 'bail', 'bail-test-4-pass.js'),
+];
+
+run({ bail: true, concurrency: 1, files }).compose(spec).compose(process.stdout);

test/fixtures/test-runner/output/bail_concurrency_1.snapshot

@@ -0,0 +1,55 @@
+✔ test 1 passes (*ms)
+✔ test 2 passes (*ms)
+✖ failing test 1 (*ms)
+Bailing out! no new test files will be started!
+✖ failing test 2 (*ms)
+ℹ tests 4
+ℹ suites 0
+ℹ pass 2
+ℹ fail 2
+ℹ cancelled 0
+ℹ skipped 0
+ℹ todo 0
+ℹ duration_ms *
+
+✖ failing tests:
+
+*
+✖ failing test 1 (*ms)
+  AssertionError [ERR_ASSERTION]: This test should fail
+
+  1 !== 2
+
+      *
+      *
+      *
+      *
+      * {
+    generatedMessage: false,
+    code: 'ERR_ASSERTION',
+    actual: 1,
+    expected: 2,
+    operator: 'strictEqual',
+    diff: 'simple'
+  }
+
+*
+✖ failing test 2 (*ms)
+  AssertionError [ERR_ASSERTION]: This test fails as well
+
+  3 !== 4
+
+      *
+      *
+      *
+      *
+      *
+      *
+      * {
+    generatedMessage: false,
+    code: 'ERR_ASSERTION',
+    actual: 3,
+    expected: 4,
+    operator: 'strictEqual',
+    diff: 'simple'
+  }