test_runner: support bail out at runner level

pmarchini · pmarchini · commit 427661d46ba6 · 2026-02-03T17:21:02.000+01:00
diff --git a/lib/internal/test_runner/harness.js b/lib/internal/test_runner/harness.js
@@ -52,6 +52,8 @@ function createTestTree(rootTestOptions, globalOptions) {
     buildSuites: [],
     isWaitingForBuildPhase: false,
     watching: false,
+    bail: globalOptions.bail,
+    bailedOut: false,
     config: globalOptions,
     coverage: null,
     resetCounters() {
diff --git a/lib/internal/test_runner/runner.js b/lib/internal/test_runner/runner.js
@@ -22,6 +22,8 @@ const {
   SafePromiseAll,
   SafePromiseAllReturnVoid,
   SafePromiseAllSettledReturnVoid,
+  SafePromisePrototypeFinally,
+  SafePromiseRace,
   SafeSet,
   StringPrototypeIndexOf,
   StringPrototypeSlice,
@@ -254,6 +256,17 @@ class FileTest extends Test {
     if (item.data.details?.error) {
       item.data.details.error = deserializeError(item.data.details.error);
     }
+
+    if (item.type === 'test:fail' && this.root.harness.bail && !this.root.harness.bailedOut) {
+      // Trigger bail if enabled and this is the first failure
+      this.root.harness.bailedOut = true;
+      this.reporter[kEmitMessage]('test:bail', {
+        __proto__: null,
+        file: this.name,
+        test: item.data,
+      });
+    }
+
     if (item.type === 'test:pass' || item.type === 'test:fail') {
       item.data.testNumber = isTopLevel ? (this.root.harness.counters.topLevel + 1) : item.data.testNumber;
       countCompletedTest({
@@ -604,6 +617,7 @@ function run(options = kEmptyObject) {
   } = options;
   const {
     concurrency,
+    bail,
     timeout,
     signal,
     files,
@@ -747,6 +761,7 @@ function run(options = kEmptyObject) {
     functionCoverage: functionCoverage,
     cwd,
     globalSetupPath,
+    bail,
   };
   const root = createTestTree(rootTestOptions, globalOptions);
   let testFiles = files ?? createTestFileList(globPatterns, cwd);
@@ -756,6 +771,16 @@ function run(options = kEmptyObject) {
     testFiles = ArrayPrototypeFilter(testFiles, (_, index) => index % shard.total === shard.index - 1);
   }
 
+  if (bail) {
+    validateBoolean(bail, 'options.bail');
+    if (watch) {
+      throw new ERR_INVALID_ARG_VALUE('options.bail', watch, 'bail not supported with watch mode');
+    }
+    if (isolation === 'none') {
+      throw new ERR_INVALID_ARG_VALUE('options.bail', isolation, 'bail not supported with \'none\' isolation');
+    }
+  }
+
   let teardown;
   let postRun;
   let filesWatcher;
@@ -770,6 +795,7 @@ function run(options = kEmptyObject) {
     hasFiles: files != null,
     globPatterns,
     only,
+    bail,
     forceExit,
     cwd,
     isolation,
@@ -792,15 +818,54 @@ function run(options = kEmptyObject) {
       teardown = () => root.harness.teardown();
     }
 
-    runFiles = () => {
-      root.harness.bootstrapPromise = null;
-      root.harness.buildPromise = null;
-      return SafePromiseAllSettledReturnVoid(testFiles, (path) => {
-        const subtest = runTestFile(path, filesWatcher, opts);
-        filesWatcher?.runningSubtests.set(path, subtest);
-        return subtest;
-      });
-    };
+    if (bail) {
+      runFiles = async () => {
+        root.harness.bootstrapPromise = null;
+        root.harness.buildPromise = null;
+
+        const running = new SafeSet();
+        let index = 0;
+
+        const shouldBail = () => bail && root.harness.bailedOut;
+
+        const enqueueNext = () => {
+          if (index < testFiles.length && !shouldBail()) {
+            const path = testFiles[index++];
+            const subtest = runTestFile(path, filesWatcher, opts);
+            filesWatcher?.runningSubtests.set(path, subtest);
+            running.add(subtest);
+            SafePromisePrototypeFinally(subtest, () => running.delete(subtest));
+          }
+        };
+
+        // Fill initial pool up to root test concurrency
+        // We use root test concurrency here because concurrency logic is handled at test level.
+        while (running.size < root.concurrency && index < testFiles.length && !shouldBail()) {
+          enqueueNext();
+        }
+
+        // As each test completes, enqueue the next one
+        while (running.size > 0) {
+          await SafePromiseRace([...running]);
+
+          // Refill pool after completion(s)
+          while (running.size < root.concurrency && index < testFiles.length && !shouldBail()) {
+            enqueueNext();
+          }
+        }
+      };
+
+    } else {
+      runFiles = () => {
+        root.harness.bootstrapPromise = null;
+        root.harness.buildPromise = null;
+        return SafePromiseAllSettledReturnVoid(testFiles, (path) => {
+          const subtest = runTestFile(path, filesWatcher, opts);
+          filesWatcher?.runningSubtests.set(path, subtest);
+          return subtest;
+        });
+      };
+    }
   } else if (isolation === 'none') {
     if (watch) {
       const absoluteTestFiles = ArrayPrototypeMap(testFiles, (file) => (isAbsolute(file) ? file : resolve(cwd, file)));
diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js
@@ -523,7 +523,7 @@ class Test extends AsyncResource {
       this.root = this;
       this.harness = options.harness;
       this.config = this.harness.config;
-      this.concurrency = 1;
+      this.concurrency = 1; // <-- is this needed?
       this.nesting = 0;
       this.only = this.config.only;
       this.reporter = new TestsStream();
@@ -540,7 +540,7 @@ class Test extends AsyncResource {
       this.root = parent.root;
       this.harness = null;
       this.config = config;
-      this.concurrency = parent.concurrency;
+      this.concurrency = parent.concurrency; // <-- is this needed?
       this.nesting = nesting;
       this.only = only;
       this.reporter = parent.reporter;
@@ -580,7 +580,7 @@ class Test extends AsyncResource {
       }
     }
 
-    switch (typeof concurrency) {
+    switch (typeof concurrency) { // <-- here we are overriding this.concurrency with the value from options!
       case 'number':
         validateUint32(concurrency, 'options.concurrency', true);
         this.concurrency = concurrency;
diff --git a/lib/internal/test_runner/tests_stream.js b/lib/internal/test_runner/tests_stream.js
@@ -149,6 +149,15 @@ class TestsStream extends Readable {
     });
   }
 
+  bail(nesting, loc, test) {
+    this[kEmitMessage]('test:bail', {
+      __proto__: null,
+      nesting,
+      test,
+      ...loc,
+    });
+  }
+
   end() {
     this.#tryPush(null);
   }
diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js
@@ -211,6 +211,7 @@ function parseCommandLine() {
   }
 
   const isTestRunner = getOptionValue('--test');
+  const bail = getOptionValue('--test-bail');
   const coverage = getOptionValue('--experimental-test-coverage');
   const forceExit = getOptionValue('--test-force-exit');
   const sourceMaps = getOptionValue('--enable-source-maps');
@@ -341,6 +342,7 @@ function parseCommandLine() {
   globalTestOptions = {
     __proto__: null,
     isTestRunner,
+    bail,
     concurrency,
     coverage,
     coverageExcludeGlobs,
diff --git a/src/node_options.cc b/src/node_options.cc
@@ -861,6 +861,12 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
             kDisallowedInEnvvar,
             false,
             OptionNamespaces::kTestRunnerNamespace);
+  AddOption("--test-bail",
+            "abort test execution after first failure",
+            &EnvironmentOptions::test_runner_bail,
+            kDisallowedInEnvvar,
+            false,
+            OptionNamespaces::kTestRunnerNamespace);
   AddOption("--test-concurrency",
             "specify test runner concurrency",
             &EnvironmentOptions::test_runner_concurrency,
diff --git a/src/node_options.h b/src/node_options.h
@@ -190,6 +190,7 @@ class EnvironmentOptions : public Options {
   std::vector<std::string> optional_env_file;
   bool has_env_file_string = false;
   bool test_runner = false;
+  bool test_runner_bail = false;
   uint64_t test_runner_concurrency = 0;
   uint64_t test_runner_timeout = 0;
   bool test_runner_coverage = false;
diff --git a/test/fixtures/test-runner/bail/bail-test-1-pass.js b/test/fixtures/test-runner/bail/bail-test-1-pass.js
@@ -0,0 +1,12 @@
+'use strict';
+const { test } = require('node:test');
+const { setTimeout } = require('timers/promises');
+
+test('test 1 passes', async () => {
+  // This should pass
+  await setTimeout(500);
+});
+
+test('test 2 passes', () => {
+  // This should pass
+});
diff --git a/test/fixtures/test-runner/bail/bail-test-2-fail.js b/test/fixtures/test-runner/bail/bail-test-2-fail.js
@@ -0,0 +1,11 @@
+'use strict';
+const { test } = require('node:test');
+const assert = require('assert');
+
+test('failing test 1', () => {
+  assert.strictEqual(1, 2, 'This test should fail');
+});
+
+test('failing test 2', () => {
+  assert.strictEqual(3, 4, 'This test should also fail but might not run');
+});
diff --git a/test/fixtures/test-runner/bail/bail-test-3-pass.js b/test/fixtures/test-runner/bail/bail-test-3-pass.js
@@ -0,0 +1,10 @@
+'use strict';
+const { test } = require('node:test');
+
+test('test 3 passes', () => {
+  // This should not run if bail happens in test 2
+});
+
+test('test 4 passes', () => {
+  // This should not run if bail happens in test 2
+});
diff --git a/test/fixtures/test-runner/bail/bail-test-4-pass.js b/test/fixtures/test-runner/bail/bail-test-4-pass.js
@@ -0,0 +1,6 @@
+'use strict';
+const { test } = require('node:test');
+
+test('test 5 passes', () => {
+  // This should not run if bail happens earlier
+});
diff --git a/test/parallel/test-runner-bail.js b/test/parallel/test-runner-bail.js
@@ -0,0 +1,114 @@
+'use strict';
+require('../common');
+const { test } = require('node:test');
+const assert = require('node:assert');
+const { spawn } = require('node:child_process');
+const { once } = require('node:events');
+const fixtures = require('../common/fixtures');
+
+const runTest = async (args, cwd = process.cwd()) => {
+  const child = spawn(process.execPath, args, {
+    cwd,
+    stdio: 'pipe',
+    env: { ...process.env },
+  });
+
+  const stdout = [];
+  const stderr = [];
+
+  child.stdout.on('data', (chunk) => stdout.push(chunk));
+  child.stderr.on('data', (chunk) => stderr.push(chunk));
+
+  const [code] = await once(child, 'exit');
+
+  return {
+    code,
+    stdout: Buffer.concat(stdout).toString('utf8'),
+    stderr: Buffer.concat(stderr).toString('utf8'),
+  };
+};
+
+test('bail stops test execution on first failure with isolation=process', async () => {
+  const fixturesDir = fixtures.path('test-runner', 'bail');
+  const result = await runTest([
+    '--test-bail',
+    '--test-concurrency=1', // <-- fixed concurrency to ensure predictable order of test execution
+    '--test',
+    'bail-test-1-pass.js',
+    'bail-test-2-fail.js',
+    'bail-test-3-pass.js',
+    'bail-test-4-pass.js',
+  ], fixturesDir);
+
+  // Should exit with non-zero code due to failure
+  assert.notStrictEqual(result.code, 0);
+
+  // bail-test-1-pass.js should run (2 tests)
+  assert.match(result.stdout, /test 1 passes/);
+  assert.match(result.stdout, /test 2 passes/);
+
+  // bail-test-2-fail.js should run and fail
+  assert.match(result.stdout, /failing test 1/);
+
+  // Files after failure should NOT run
+  assert.doesNotMatch(result.stdout, /test 3 passes/);
+  assert.doesNotMatch(result.stdout, /test 4 passes/);
+  assert.doesNotMatch(result.stdout, /test 5 passes/);
+});
+
+test.todo('bail stops test execution on first failure with isolation=none');
+
+test('bail is not supported with isolation=none', async () => {
+  const fixturesDir = fixtures.path('test-runner', 'bail');
+  const result = await runTest([
+    '--test',
+    '--test-bail',
+    '--test-isolation=none',
+    'bail-test-1-pass.js',
+  ], fixturesDir);
+
+  // Should exit with error
+  assert.notStrictEqual(result.code, 0);
+
+  // Should contain error message about bail+isolation incompatibility
+  assert.match(result.stderr, /bail not supported with 'none' isolation/);
+});
+
+test('bail throws error when combined with watch mode', async () => {
+  const fixturesDir = fixtures.path('test-runner', 'bail');
+  const result = await runTest([
+    '--test',
+    '--test-bail',
+    '--watch',
+    'bail-test-1-pass.js',
+  ], fixturesDir);
+
+  // Should exit with error
+  assert.notStrictEqual(result.code, 0);
+
+  // Should contain error message about bail+watch incompatibility
+  assert.match(result.stderr, /not supported with watch mode/);
+});
+
+test('without bail, all tests run even after failure', async () => {
+  const fixturesDir = fixtures.path('test-runner', 'bail');
+  const result = await runTest([
+    '--test',
+    'bail-test-1-pass.js',
+    'bail-test-2-fail.js',
+    'bail-test-3-pass.js',
+  ], fixturesDir);
+
+  // Should exit with non-zero code due to failure
+  assert.notStrictEqual(result.code, 0);
+
+  // All tests should run
+  assert.match(result.stdout, /test 1 passes/);
+  assert.match(result.stdout, /test 2 passes/);
+  assert.match(result.stdout, /failing test 1/);
+  assert.match(result.stdout, /test 3 passes/);
+  assert.match(result.stdout, /test 4 passes/);
+
+  // Should NOT contain bail event
+  assert.doesNotMatch(result.stdout, /test:bail/);
+});
diff --git a/test/parallel/test-runner-run-bail-isolation-none.mjs b/test/parallel/test-runner-run-bail-isolation-none.mjs
@@ -0,0 +1,15 @@
+import '../common/index.mjs';
+import assert from 'node:assert';
+import { run } from 'node:test';
+import * as fixtures from '../common/fixtures.mjs';
+
+const files = [
+  fixtures.path('test-runner', 'bail', 'bail-test-1-pass.js'),
+];
+
+assert.throws(() => {
+  run({ bail: true, isolation: 'none', files });
+}, {
+  code: 'ERR_INVALID_ARG_VALUE',
+  message: /bail not supported with 'none' isolation/,
+});
diff --git a/test/parallel/test-runner-run-bail.mjs b/test/parallel/test-runner-run-bail.mjs
