From 533105ffb9d39a1d2d730f26ec3513f4737d3034 Mon Sep 17 00:00:00 2001 From: matt-halliday Date: Mon, 18 Aug 2025 11:22:06 +0100 Subject: [PATCH 1/6] fix: sharding in combination with the --url cli option --- src/config/jest-playwright.ts | 2 + src/config/jest-sequencer.test.ts | 303 ++++++++++++++++++++++++++++++ src/config/jest-sequencer.ts | 42 +++++ src/test-storybook.ts | 4 +- tsup.config.ts | 2 +- 5 files changed, 351 insertions(+), 2 deletions(-) create mode 100644 src/config/jest-sequencer.test.ts create mode 100644 src/config/jest-sequencer.ts diff --git a/src/config/jest-playwright.ts b/src/config/jest-playwright.ts index 4fab6d3f..a80d009a 100644 --- a/src/config/jest-playwright.ts +++ b/src/config/jest-playwright.ts @@ -1,3 +1,4 @@ +/** @jest-config-loader-options {"transpileOnly": true} */ import path from 'path'; import { getProjectRoot } from 'storybook/internal/common'; import type { Config } from '@jest/types'; @@ -95,6 +96,7 @@ export const getJestConfig = (): Config.InitialOptions => { exitOnPageError: false, }, }, + testSequencer: require.resolve(`./config/jest-sequencer`), watchPlugins: [ require.resolve('jest-watch-typeahead/filename'), require.resolve('jest-watch-typeahead/testname'), diff --git a/src/config/jest-sequencer.test.ts b/src/config/jest-sequencer.test.ts new file mode 100644 index 00000000..5c0ca7e1 --- /dev/null +++ b/src/config/jest-sequencer.test.ts @@ -0,0 +1,303 @@ +import StorybookTestSequencer from './jest-sequencer'; +import type { Test } from '@jest/test-result'; + +// Mock test data factory +const createMockTest = (path: string): Test => ({ + context: { + config: { + displayName: undefined, + rootDir: '/mock/root', + }, + hasteFS: {}, + moduleMap: {}, + resolver: {}, + }, + duration: undefined, + path, +}) as Test; + +describe('StorybookTestSequencer', () => { + let sequencer: StorybookTestSequencer; + + beforeEach(() => { + sequencer = new StorybookTestSequencer(); + }); + + describe('sort', () => { + it('should sort tests alphabetically by path', () => { + const tests = [ + createMockTest('/path/to/z-test.js'), + createMockTest('/path/to/a-test.js'), + createMockTest('/path/to/m-test.js'), + ]; + + const result = sequencer.sort(tests); + + expect(result.map(test => test.path)).toEqual([ + '/path/to/a-test.js', + '/path/to/m-test.js', + '/path/to/z-test.js', + ]); + }); + + it('should handle empty test array', () => { + const tests: Test[] = []; + const result = sequencer.sort(tests); + expect(result).toEqual([]); + }); + + it('should handle single test', () => { + const tests = [createMockTest('/path/to/single-test.js')]; + const result = sequencer.sort(tests); + expect(result).toEqual(tests); + }); + + it('should normalize paths for index JSON tests', () => { + const tests = [ + createMockTest('/path/to/test-storybook-index-json__1234567890/story-z.test.js'), + createMockTest('/different/path/test-storybook-index-json__abcdefghijk/story-a.test.js'), + createMockTest('/another/path/test-storybook-index-json__xyz9876543/story-m.test.js'), + ]; + + const result = sequencer.sort(tests); + + // After sorting by basename, original paths should be preserved but in sorted order + expect(result.map(test => test.path)).toEqual([ + '/different/path/test-storybook-index-json__abcdefghijk/story-a.test.js', + '/another/path/test-storybook-index-json__xyz9876543/story-m.test.js', + '/path/to/test-storybook-index-json__1234567890/story-z.test.js', + ]); + }); + }); + + describe('shard', () => { + it('should divide tests into shards correctly', () => { + const tests = [ + createMockTest('/path/to/a-test.js'), + createMockTest('/path/to/b-test.js'), + createMockTest('/path/to/c-test.js'), + createMockTest('/path/to/d-test.js'), + createMockTest('/path/to/e-test.js'), + createMockTest('/path/to/f-test.js'), + ]; + + // Test first shard (1 of 3) + const shard1 = sequencer.shard(tests, { shardIndex: 1, shardCount: 3 }); + expect(shard1.length).toBe(2); + expect(shard1.map(test => test.path)).toEqual([ + '/path/to/a-test.js', + '/path/to/b-test.js', + ]); + + // Test second shard (2 of 3) + const shard2 = sequencer.shard(tests, { shardIndex: 2, shardCount: 3 }); + expect(shard2.length).toBe(2); + expect(shard2.map(test => test.path)).toEqual([ + '/path/to/c-test.js', + '/path/to/d-test.js', + ]); + + // Test third shard (3 of 3) + const shard3 = sequencer.shard(tests, { shardIndex: 3, shardCount: 3 }); + expect(shard3.length).toBe(2); + expect(shard3.map(test => test.path)).toEqual([ + '/path/to/e-test.js', + '/path/to/f-test.js', + ]); + }); + + it('should handle uneven division of tests', () => { + const tests = [ + createMockTest('/path/to/a-test.js'), + createMockTest('/path/to/b-test.js'), + createMockTest('/path/to/c-test.js'), + createMockTest('/path/to/d-test.js'), + createMockTest('/path/to/e-test.js'), + ]; + + // With 5 tests and 3 shards: shard sizes should be [2, 2, 1] + const shard1 = sequencer.shard(tests, { shardIndex: 1, shardCount: 3 }); + expect(shard1.length).toBe(2); + + const shard2 = sequencer.shard(tests, { shardIndex: 2, shardCount: 3 }); + expect(shard2.length).toBe(2); + + const shard3 = sequencer.shard(tests, { shardIndex: 3, shardCount: 3 }); + expect(shard3.length).toBe(1); + }); + + it('should handle single shard (no sharding)', () => { + const tests = [ + createMockTest('/path/to/z-test.js'), + createMockTest('/path/to/a-test.js'), + createMockTest('/path/to/m-test.js'), + ]; + + const result = sequencer.shard(tests, { shardIndex: 1, shardCount: 1 }); + + expect(result.length).toBe(3); + expect(result.map(test => test.path)).toEqual([ + '/path/to/a-test.js', + '/path/to/m-test.js', + '/path/to/z-test.js', + ]); + }); + + it('should sort tests before sharding', () => { + const tests = [ + createMockTest('/path/to/z-test.js'), + createMockTest('/path/to/a-test.js'), + createMockTest('/path/to/m-test.js'), + createMockTest('/path/to/b-test.js'), + ]; + + const shard1 = sequencer.shard(tests, { shardIndex: 1, shardCount: 2 }); + const shard2 = sequencer.shard(tests, { shardIndex: 2, shardCount: 2 }); + + // First shard should get first 2 alphabetically sorted tests + expect(shard1.map(test => test.path)).toEqual([ + '/path/to/a-test.js', + '/path/to/b-test.js', + ]); + + // Second shard should get last 2 alphabetically sorted tests + expect(shard2.map(test => test.path)).toEqual([ + '/path/to/m-test.js', + '/path/to/z-test.js', + ]); + }); + + it('should normalize paths for index JSON tests before sharding', () => { + const tests = [ + createMockTest('/different/path/test-storybook-index-json__xyz123/story-z.test.js'), + createMockTest('/another/path/test-storybook-index-json__abc789/story-a.test.js'), + createMockTest('/path/to/test-storybook-index-json__def456/story-m.test.js'), + createMockTest('/final/path/test-storybook-index-json__ghi012/story-b.test.js'), + ]; + + const shard1 = sequencer.shard(tests, { shardIndex: 1, shardCount: 2 }); + const shard2 = sequencer.shard(tests, { shardIndex: 2, shardCount: 2 }); + + // After sorting by basename but preserving original paths + expect(shard1.map(test => test.path)).toEqual([ + '/another/path/test-storybook-index-json__abc789/story-a.test.js', + '/final/path/test-storybook-index-json__ghi012/story-b.test.js', + ]); + + expect(shard2.map(test => test.path)).toEqual([ + '/path/to/test-storybook-index-json__def456/story-m.test.js', + '/different/path/test-storybook-index-json__xyz123/story-z.test.js', + ]); + }); + + it('should handle empty test array for sharding', () => { + const tests: Test[] = []; + const result = sequencer.shard(tests, { shardIndex: 1, shardCount: 2 }); + expect(result).toEqual([]); + }); + + it('should handle out-of-bounds shard index', () => { + const tests = [ + createMockTest('/path/to/a-test.js'), + createMockTest('/path/to/b-test.js'), + ]; + + // Request shard 3 of 2 (out of bounds) + const result = sequencer.shard(tests, { shardIndex: 3, shardCount: 2 }); + expect(result).toEqual([]); + }); + }); + + describe('normalizeTestPathsForIndexJSON integration', () => { + it('should normalize when ANY test contains index JSON pattern', () => { + const allIndexTests = [ + createMockTest('/path/to/test-storybook-index-json__temp1/story-a.test.js'), + createMockTest('/different/test-storybook-index-json__temp2/story-b.test.js'), + ]; + + const mixedTests = [ + createMockTest('/path/to/test-storybook-index-json__temp3/story-a.test.js'), + createMockTest('/path/to/regular-test.js'), + ]; + + const regularTests = [ + createMockTest('/path/to/regular-test-a.js'), + createMockTest('/path/to/regular-test-b.js'), + ]; + + // All index tests should be sorted by basename but preserve original paths + const allIndexResult = sequencer.sort(allIndexTests); + expect(allIndexResult.map(test => test.path)).toEqual([ + '/path/to/test-storybook-index-json__temp1/story-a.test.js', + '/different/test-storybook-index-json__temp2/story-b.test.js', + ]); + + // Mixed tests should also sort by basename (since ANY test has index JSON pattern) + const mixedResult = sequencer.sort(mixedTests); + expect(mixedResult.map(test => test.path)).toEqual([ + '/path/to/regular-test.js', + '/path/to/test-storybook-index-json__temp3/story-a.test.js', + ]); + + // Regular tests should sort by full path + const regularResult = sequencer.sort(regularTests); + expect(regularResult.map(test => test.path)).toEqual([ + '/path/to/regular-test-a.js', + '/path/to/regular-test-b.js', + ]); + }); + + it('should sort by basename when ANY test contains index JSON pattern while preserving original paths', () => { + const allIndexTests = [ + createMockTest('/path/to/test-storybook-index-json__temp1/story-b.test.js'), + createMockTest('/different/test-storybook-index-json__temp2/story-a.test.js'), + ]; + + const mixedTests = [ + createMockTest('/path/to/test-storybook-index-json__temp3/story-z.test.js'), + createMockTest('/path/to/regular-test.js'), + ]; + + const regularTests = [ + createMockTest('/path/to/regular-test-b.js'), + createMockTest('/path/to/regular-test-a.js'), + ]; + + // All index tests should be sorted by basename but preserve original paths + const allIndexResult = sequencer.sort(allIndexTests); + expect(allIndexResult.map(test => test.path)).toEqual([ + '/different/test-storybook-index-json__temp2/story-a.test.js', + '/path/to/test-storybook-index-json__temp1/story-b.test.js', + ]); + + // Mixed tests should also sort by basename (since ANY test has index JSON pattern) + const mixedResult = sequencer.sort(mixedTests); + expect(mixedResult.map(test => test.path)).toEqual([ + '/path/to/regular-test.js', + '/path/to/test-storybook-index-json__temp3/story-z.test.js', + ]); + + // Regular tests should sort by full path + const regularResult = sequencer.sort(regularTests); + expect(regularResult.map(test => test.path)).toEqual([ + '/path/to/regular-test-a.js', + '/path/to/regular-test-b.js', + ]); + }); + + it('should handle mixed index JSON and regular tests (normalization applies when any test is index JSON)', () => { + const tests = [ + createMockTest('/path/to/regular-test.js'), + createMockTest('/path/to/test-storybook-index-json__temp123/story.test.js'), + ]; + + const result = sequencer.sort(tests); + + // Since ANY test contains index JSON pattern, sorting uses basename comparison but preserves original paths + expect(result.map(test => test.path)).toEqual([ + '/path/to/regular-test.js', + '/path/to/test-storybook-index-json__temp123/story.test.js', + ]); + }); + }); +}); diff --git a/src/config/jest-sequencer.ts b/src/config/jest-sequencer.ts new file mode 100644 index 00000000..55f84b4d --- /dev/null +++ b/src/config/jest-sequencer.ts @@ -0,0 +1,42 @@ +import Sequencer, { ShardOptions } from '@jest/test-sequencer'; +import type { Test } from '@jest/test-result'; +import { basename } from 'path'; + +/** + * Sorts tests using basename for comparison when running against index JSON stories. + * When tests include 'test-storybook-index-json__', this function sorts the tests + * by their basename to ensure consistent ordering across different temporary directories, + * while preserving the original test paths. + * + * @param tests - Array of Jest test objects + * @returns Array of tests sorted by basename when index JSON pattern is detected, otherwise sorted by full path + */ +const sortForIndexJSON = (tests: Array): Array => { + const isIndexJSON = tests.some((test) => test.path.includes('test-storybook-index-json__')); + + return tests.sort((a, b) => { + const pathA = isIndexJSON ? basename(a.path) : a.path; + const pathB = isIndexJSON ? basename(b.path) : b.path; + return pathA > pathB ? 1 : -1; + }); +}; + +/** + * Custom Jest sequencer for Storybook tests that ensures consistent + * test ordering and proper sharding support for distributed test execution. + */ +class StorybookTestSequencer extends Sequencer { + shard(tests: Array, { shardIndex, shardCount }: ShardOptions) { + const shardSize = Math.ceil(tests.length / shardCount); + const shardStart = shardSize * (shardIndex - 1); + const shardEnd = shardSize * shardIndex; + + return sortForIndexJSON(tests).slice(shardStart, shardEnd); + } + + sort(tests: Array) { + return sortForIndexJSON(tests); + } +} + +export default StorybookTestSequencer; diff --git a/src/test-storybook.ts b/src/test-storybook.ts index 8004e530..296fde2b 100644 --- a/src/test-storybook.ts +++ b/src/test-storybook.ts @@ -214,7 +214,9 @@ async function getIndexTempDir(url: string) { const indexJson = await getIndexJson(url); const titleIdToTest = transformPlaywrightJson(indexJson); - tmpDir = tempy.directory(); + tmpDir = tempy.directory({ + prefix: 'test-storybook-index-json__', + }); for (const [titleId, test] of Object.entries(titleIdToTest)) { const tmpFile = path.join(tmpDir, `${titleId}.test.js`); fs.writeFileSync(tmpFile, test); diff --git a/tsup.config.ts b/tsup.config.ts index f0474e64..88f8f4a5 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'tsup'; export default defineConfig([ { clean: true, - entry: ['./src/index.ts', './src/test-storybook.ts'], + entry: ['./src/index.ts', './src/test-storybook.ts', './src/config/jest-sequencer.ts'], format: ['cjs', 'esm'], splitting: false, dts: true, From c0d00ee55c42c1094f16b2e683860067ef4e2255 Mon Sep 17 00:00:00 2001 From: matt-halliday Date: Tue, 19 Aug 2025 09:32:22 +0100 Subject: [PATCH 2/6] feat: simplify approach --- src/config/jest-filename-sequencer.test.ts | 182 +++++++++++++ src/config/jest-filename-sequencer.ts | 23 ++ src/config/jest-playwright.ts | 3 +- src/config/jest-sequencer.test.ts | 303 --------------------- src/config/jest-sequencer.ts | 42 --- src/test-storybook.ts | 5 +- tsup.config.ts | 2 +- 7 files changed, 210 insertions(+), 350 deletions(-) create mode 100644 src/config/jest-filename-sequencer.test.ts create mode 100644 src/config/jest-filename-sequencer.ts delete mode 100644 src/config/jest-sequencer.test.ts delete mode 100644 src/config/jest-sequencer.ts diff --git a/src/config/jest-filename-sequencer.test.ts b/src/config/jest-filename-sequencer.test.ts new file mode 100644 index 00000000..7c8c2be2 --- /dev/null +++ b/src/config/jest-filename-sequencer.test.ts @@ -0,0 +1,182 @@ +import FilenameSortedTestSequencer from './jest-filename-sequencer'; +import type { Test } from '@jest/test-result'; + +// Mock test data factory +const createMockTest = (path: string): Test => ({ + context: { + config: { + displayName: undefined, + rootDir: '/mock/root', + }, + hasteFS: {}, + moduleMap: {}, + resolver: {}, + }, + duration: undefined, + path, +}) as Test; + +describe('FilenameSortedTestSequencer', () => { + let sequencer: FilenameSortedTestSequencer; + + beforeEach(() => { + sequencer = new FilenameSortedTestSequencer(); + }); + + describe('sort', () => { + it('should sort tests by basename (filename)', () => { + const tests = [ + createMockTest('/path/to/story-z.test.js'), + createMockTest('/different/path/story-a.test.js'), + createMockTest('/another/path/story-m.test.js'), + ]; + + const result = sequencer.sort(tests); + + expect(result.map(test => test.path)).toEqual([ + '/different/path/story-a.test.js', + '/another/path/story-m.test.js', + '/path/to/story-z.test.js', + ]); + }); + + it('should handle tests with same basename from different directories', () => { + const tests = [ + createMockTest('/components/button/Button.test.js'), + createMockTest('/pages/home/Button.test.js'), + createMockTest('/utils/helpers/Button.test.js'), + ]; + + const result = sequencer.sort(tests); + + expect(result).toHaveLength(3); + expect(result.every(test => test.path.endsWith('Button.test.js'))).toBe(true); + }); + + it('should handle mixed file extensions', () => { + const tests = [ + createMockTest('/path/component.test.tsx'), + createMockTest('/path/component.test.js'), + createMockTest('/path/component.test.ts'), + ]; + + const result = sequencer.sort(tests); + + expect(result.map(test => test.path)).toEqual([ + '/path/component.test.js', + '/path/component.test.ts', + '/path/component.test.tsx', + ]); + }); + + it('should handle empty test array', () => { + const tests: Test[] = []; + const result = sequencer.sort(tests); + expect(result).toEqual([]); + }); + + it('should handle single test', () => { + const tests = [createMockTest('/path/to/single.test.js')]; + const result = sequencer.sort(tests); + expect(result).toEqual(tests); + }); + + it('should sort complex story filenames correctly', () => { + const tests = [ + createMockTest('/stories/Button.stories.test.js'), + createMockTest('/stories/Alert.stories.test.js'), + createMockTest('/stories/Modal.stories.test.js'), + createMockTest('/stories/Card.stories.test.js'), + ]; + + const result = sequencer.sort(tests); + + expect(result.map(test => test.path)).toEqual([ + '/stories/Alert.stories.test.js', + '/stories/Button.stories.test.js', + '/stories/Card.stories.test.js', + '/stories/Modal.stories.test.js', + ]); + }); + }); + + describe('shard', () => { + it('should sort tests first, then divide into shards correctly', () => { + const tests = [ + createMockTest('/path/f-test.js'), + createMockTest('/path/a-test.js'), + createMockTest('/path/d-test.js'), + createMockTest('/path/b-test.js'), + createMockTest('/path/e-test.js'), + createMockTest('/path/c-test.js'), + ]; + + // Test first shard (1 of 3) + const shard1 = sequencer.shard(tests, { shardIndex: 1, shardCount: 3 }); + expect(shard1.length).toBe(2); + expect(shard1.map(test => test.path)).toEqual([ + '/path/a-test.js', + '/path/b-test.js', + ]); + + // Test second shard (2 of 3) + const shard2 = sequencer.shard(tests, { shardIndex: 2, shardCount: 3 }); + expect(shard2.length).toBe(2); + expect(shard2.map(test => test.path)).toEqual([ + '/path/c-test.js', + '/path/d-test.js', + ]); + + // Test third shard (3 of 3) + const shard3 = sequencer.shard(tests, { shardIndex: 3, shardCount: 3 }); + expect(shard3.length).toBe(2); + expect(shard3.map(test => test.path)).toEqual([ + '/path/e-test.js', + '/path/f-test.js', + ]); + }); + + it('should handle uneven shard distribution', () => { + const tests = [ + createMockTest('/path/c-test.js'), + createMockTest('/path/a-test.js'), + createMockTest('/path/b-test.js'), + ]; + + const shard1 = sequencer.shard(tests, { shardIndex: 1, shardCount: 2 }); + expect(shard1.length).toBe(2); + expect(shard1.map(test => test.path)).toEqual([ + '/path/a-test.js', + '/path/b-test.js', + ]); + + const shard2 = sequencer.shard(tests, { shardIndex: 2, shardCount: 2 }); + expect(shard2.length).toBe(1); + expect(shard2.map(test => test.path)).toEqual([ + '/path/c-test.js', + ]); + }); + + it('should handle empty test array', () => { + const tests: Test[] = []; + const result = sequencer.shard(tests, { shardIndex: 1, shardCount: 2 }); + expect(result).toEqual([]); + }); + + it('should handle single shard', () => { + const tests = [ + createMockTest('/path/c-test.js'), + createMockTest('/path/a-test.js'), + createMockTest('/path/b-test.js'), + ]; + + const result = sequencer.shard(tests, { shardIndex: 1, shardCount: 1 }); + + expect(result.map(test => test.path)).toEqual([ + '/path/a-test.js', + '/path/b-test.js', + '/path/c-test.js', + ]); + }); + }); +}); diff --git a/src/config/jest-filename-sequencer.ts b/src/config/jest-filename-sequencer.ts new file mode 100644 index 00000000..1806bc4c --- /dev/null +++ b/src/config/jest-filename-sequencer.ts @@ -0,0 +1,23 @@ +import Sequencer, { ShardOptions } from '@jest/test-sequencer'; +import type { Test } from '@jest/test-result'; +import { basename } from 'path'; + +const sortByFilename = (tests: Array): Array => { + return tests.sort((a, b) => basename(a.path).localeCompare(basename(b.path))); +}; + +class FilenameSortedTestSequencer extends Sequencer { + shard(tests: Array, { shardIndex, shardCount }: ShardOptions) { + const shardSize = Math.ceil(tests.length / shardCount); + const shardStart = shardSize * (shardIndex - 1); + const shardEnd = shardSize * shardIndex; + + return sortByFilename(tests).slice(shardStart, shardEnd); + } + + sort(tests: Array) { + return sortByFilename(tests); + } +} + +export default FilenameSortedTestSequencer; diff --git a/src/config/jest-playwright.ts b/src/config/jest-playwright.ts index a80d009a..c459a858 100644 --- a/src/config/jest-playwright.ts +++ b/src/config/jest-playwright.ts @@ -50,6 +50,7 @@ export const getJestConfig = (): Config.InitialOptions => { TEST_BROWSERS, STORYBOOK_COLLECT_COVERAGE, STORYBOOK_JUNIT, + TEST_INDEX_JSON, } = process.env; const jestJunitPath = path.dirname( @@ -96,7 +97,7 @@ export const getJestConfig = (): Config.InitialOptions => { exitOnPageError: false, }, }, - testSequencer: require.resolve(`./config/jest-sequencer`), + testSequencer: TEST_INDEX_JSON ? require.resolve(`./config/jest-filename-sequencer`) : undefined, watchPlugins: [ require.resolve('jest-watch-typeahead/filename'), require.resolve('jest-watch-typeahead/testname'), diff --git a/src/config/jest-sequencer.test.ts b/src/config/jest-sequencer.test.ts deleted file mode 100644 index 5c0ca7e1..00000000 --- a/src/config/jest-sequencer.test.ts +++ /dev/null @@ -1,303 +0,0 @@ -import StorybookTestSequencer from './jest-sequencer'; -import type { Test } from '@jest/test-result'; - -// Mock test data factory -const createMockTest = (path: string): Test => ({ - context: { - config: { - displayName: undefined, - rootDir: '/mock/root', - }, - hasteFS: {}, - moduleMap: {}, - resolver: {}, - }, - duration: undefined, - path, -}) as Test; - -describe('StorybookTestSequencer', () => { - let sequencer: StorybookTestSequencer; - - beforeEach(() => { - sequencer = new StorybookTestSequencer(); - }); - - describe('sort', () => { - it('should sort tests alphabetically by path', () => { - const tests = [ - createMockTest('/path/to/z-test.js'), - createMockTest('/path/to/a-test.js'), - createMockTest('/path/to/m-test.js'), - ]; - - const result = sequencer.sort(tests); - - expect(result.map(test => test.path)).toEqual([ - '/path/to/a-test.js', - '/path/to/m-test.js', - '/path/to/z-test.js', - ]); - }); - - it('should handle empty test array', () => { - const tests: Test[] = []; - const result = sequencer.sort(tests); - expect(result).toEqual([]); - }); - - it('should handle single test', () => { - const tests = [createMockTest('/path/to/single-test.js')]; - const result = sequencer.sort(tests); - expect(result).toEqual(tests); - }); - - it('should normalize paths for index JSON tests', () => { - const tests = [ - createMockTest('/path/to/test-storybook-index-json__1234567890/story-z.test.js'), - createMockTest('/different/path/test-storybook-index-json__abcdefghijk/story-a.test.js'), - createMockTest('/another/path/test-storybook-index-json__xyz9876543/story-m.test.js'), - ]; - - const result = sequencer.sort(tests); - - // After sorting by basename, original paths should be preserved but in sorted order - expect(result.map(test => test.path)).toEqual([ - '/different/path/test-storybook-index-json__abcdefghijk/story-a.test.js', - '/another/path/test-storybook-index-json__xyz9876543/story-m.test.js', - '/path/to/test-storybook-index-json__1234567890/story-z.test.js', - ]); - }); - }); - - describe('shard', () => { - it('should divide tests into shards correctly', () => { - const tests = [ - createMockTest('/path/to/a-test.js'), - createMockTest('/path/to/b-test.js'), - createMockTest('/path/to/c-test.js'), - createMockTest('/path/to/d-test.js'), - createMockTest('/path/to/e-test.js'), - createMockTest('/path/to/f-test.js'), - ]; - - // Test first shard (1 of 3) - const shard1 = sequencer.shard(tests, { shardIndex: 1, shardCount: 3 }); - expect(shard1.length).toBe(2); - expect(shard1.map(test => test.path)).toEqual([ - '/path/to/a-test.js', - '/path/to/b-test.js', - ]); - - // Test second shard (2 of 3) - const shard2 = sequencer.shard(tests, { shardIndex: 2, shardCount: 3 }); - expect(shard2.length).toBe(2); - expect(shard2.map(test => test.path)).toEqual([ - '/path/to/c-test.js', - '/path/to/d-test.js', - ]); - - // Test third shard (3 of 3) - const shard3 = sequencer.shard(tests, { shardIndex: 3, shardCount: 3 }); - expect(shard3.length).toBe(2); - expect(shard3.map(test => test.path)).toEqual([ - '/path/to/e-test.js', - '/path/to/f-test.js', - ]); - }); - - it('should handle uneven division of tests', () => { - const tests = [ - createMockTest('/path/to/a-test.js'), - createMockTest('/path/to/b-test.js'), - createMockTest('/path/to/c-test.js'), - createMockTest('/path/to/d-test.js'), - createMockTest('/path/to/e-test.js'), - ]; - - // With 5 tests and 3 shards: shard sizes should be [2, 2, 1] - const shard1 = sequencer.shard(tests, { shardIndex: 1, shardCount: 3 }); - expect(shard1.length).toBe(2); - - const shard2 = sequencer.shard(tests, { shardIndex: 2, shardCount: 3 }); - expect(shard2.length).toBe(2); - - const shard3 = sequencer.shard(tests, { shardIndex: 3, shardCount: 3 }); - expect(shard3.length).toBe(1); - }); - - it('should handle single shard (no sharding)', () => { - const tests = [ - createMockTest('/path/to/z-test.js'), - createMockTest('/path/to/a-test.js'), - createMockTest('/path/to/m-test.js'), - ]; - - const result = sequencer.shard(tests, { shardIndex: 1, shardCount: 1 }); - - expect(result.length).toBe(3); - expect(result.map(test => test.path)).toEqual([ - '/path/to/a-test.js', - '/path/to/m-test.js', - '/path/to/z-test.js', - ]); - }); - - it('should sort tests before sharding', () => { - const tests = [ - createMockTest('/path/to/z-test.js'), - createMockTest('/path/to/a-test.js'), - createMockTest('/path/to/m-test.js'), - createMockTest('/path/to/b-test.js'), - ]; - - const shard1 = sequencer.shard(tests, { shardIndex: 1, shardCount: 2 }); - const shard2 = sequencer.shard(tests, { shardIndex: 2, shardCount: 2 }); - - // First shard should get first 2 alphabetically sorted tests - expect(shard1.map(test => test.path)).toEqual([ - '/path/to/a-test.js', - '/path/to/b-test.js', - ]); - - // Second shard should get last 2 alphabetically sorted tests - expect(shard2.map(test => test.path)).toEqual([ - '/path/to/m-test.js', - '/path/to/z-test.js', - ]); - }); - - it('should normalize paths for index JSON tests before sharding', () => { - const tests = [ - createMockTest('/different/path/test-storybook-index-json__xyz123/story-z.test.js'), - createMockTest('/another/path/test-storybook-index-json__abc789/story-a.test.js'), - createMockTest('/path/to/test-storybook-index-json__def456/story-m.test.js'), - createMockTest('/final/path/test-storybook-index-json__ghi012/story-b.test.js'), - ]; - - const shard1 = sequencer.shard(tests, { shardIndex: 1, shardCount: 2 }); - const shard2 = sequencer.shard(tests, { shardIndex: 2, shardCount: 2 }); - - // After sorting by basename but preserving original paths - expect(shard1.map(test => test.path)).toEqual([ - '/another/path/test-storybook-index-json__abc789/story-a.test.js', - '/final/path/test-storybook-index-json__ghi012/story-b.test.js', - ]); - - expect(shard2.map(test => test.path)).toEqual([ - '/path/to/test-storybook-index-json__def456/story-m.test.js', - '/different/path/test-storybook-index-json__xyz123/story-z.test.js', - ]); - }); - - it('should handle empty test array for sharding', () => { - const tests: Test[] = []; - const result = sequencer.shard(tests, { shardIndex: 1, shardCount: 2 }); - expect(result).toEqual([]); - }); - - it('should handle out-of-bounds shard index', () => { - const tests = [ - createMockTest('/path/to/a-test.js'), - createMockTest('/path/to/b-test.js'), - ]; - - // Request shard 3 of 2 (out of bounds) - const result = sequencer.shard(tests, { shardIndex: 3, shardCount: 2 }); - expect(result).toEqual([]); - }); - }); - - describe('normalizeTestPathsForIndexJSON integration', () => { - it('should normalize when ANY test contains index JSON pattern', () => { - const allIndexTests = [ - createMockTest('/path/to/test-storybook-index-json__temp1/story-a.test.js'), - createMockTest('/different/test-storybook-index-json__temp2/story-b.test.js'), - ]; - - const mixedTests = [ - createMockTest('/path/to/test-storybook-index-json__temp3/story-a.test.js'), - createMockTest('/path/to/regular-test.js'), - ]; - - const regularTests = [ - createMockTest('/path/to/regular-test-a.js'), - createMockTest('/path/to/regular-test-b.js'), - ]; - - // All index tests should be sorted by basename but preserve original paths - const allIndexResult = sequencer.sort(allIndexTests); - expect(allIndexResult.map(test => test.path)).toEqual([ - '/path/to/test-storybook-index-json__temp1/story-a.test.js', - '/different/test-storybook-index-json__temp2/story-b.test.js', - ]); - - // Mixed tests should also sort by basename (since ANY test has index JSON pattern) - const mixedResult = sequencer.sort(mixedTests); - expect(mixedResult.map(test => test.path)).toEqual([ - '/path/to/regular-test.js', - '/path/to/test-storybook-index-json__temp3/story-a.test.js', - ]); - - // Regular tests should sort by full path - const regularResult = sequencer.sort(regularTests); - expect(regularResult.map(test => test.path)).toEqual([ - '/path/to/regular-test-a.js', - '/path/to/regular-test-b.js', - ]); - }); - - it('should sort by basename when ANY test contains index JSON pattern while preserving original paths', () => { - const allIndexTests = [ - createMockTest('/path/to/test-storybook-index-json__temp1/story-b.test.js'), - createMockTest('/different/test-storybook-index-json__temp2/story-a.test.js'), - ]; - - const mixedTests = [ - createMockTest('/path/to/test-storybook-index-json__temp3/story-z.test.js'), - createMockTest('/path/to/regular-test.js'), - ]; - - const regularTests = [ - createMockTest('/path/to/regular-test-b.js'), - createMockTest('/path/to/regular-test-a.js'), - ]; - - // All index tests should be sorted by basename but preserve original paths - const allIndexResult = sequencer.sort(allIndexTests); - expect(allIndexResult.map(test => test.path)).toEqual([ - '/different/test-storybook-index-json__temp2/story-a.test.js', - '/path/to/test-storybook-index-json__temp1/story-b.test.js', - ]); - - // Mixed tests should also sort by basename (since ANY test has index JSON pattern) - const mixedResult = sequencer.sort(mixedTests); - expect(mixedResult.map(test => test.path)).toEqual([ - '/path/to/regular-test.js', - '/path/to/test-storybook-index-json__temp3/story-z.test.js', - ]); - - // Regular tests should sort by full path - const regularResult = sequencer.sort(regularTests); - expect(regularResult.map(test => test.path)).toEqual([ - '/path/to/regular-test-a.js', - '/path/to/regular-test-b.js', - ]); - }); - - it('should handle mixed index JSON and regular tests (normalization applies when any test is index JSON)', () => { - const tests = [ - createMockTest('/path/to/regular-test.js'), - createMockTest('/path/to/test-storybook-index-json__temp123/story.test.js'), - ]; - - const result = sequencer.sort(tests); - - // Since ANY test contains index JSON pattern, sorting uses basename comparison but preserves original paths - expect(result.map(test => test.path)).toEqual([ - '/path/to/regular-test.js', - '/path/to/test-storybook-index-json__temp123/story.test.js', - ]); - }); - }); -}); diff --git a/src/config/jest-sequencer.ts b/src/config/jest-sequencer.ts deleted file mode 100644 index 55f84b4d..00000000 --- a/src/config/jest-sequencer.ts +++ /dev/null @@ -1,42 +0,0 @@ -import Sequencer, { ShardOptions } from '@jest/test-sequencer'; -import type { Test } from '@jest/test-result'; -import { basename } from 'path'; - -/** - * Sorts tests using basename for comparison when running against index JSON stories. - * When tests include 'test-storybook-index-json__', this function sorts the tests - * by their basename to ensure consistent ordering across different temporary directories, - * while preserving the original test paths. - * - * @param tests - Array of Jest test objects - * @returns Array of tests sorted by basename when index JSON pattern is detected, otherwise sorted by full path - */ -const sortForIndexJSON = (tests: Array): Array => { - const isIndexJSON = tests.some((test) => test.path.includes('test-storybook-index-json__')); - - return tests.sort((a, b) => { - const pathA = isIndexJSON ? basename(a.path) : a.path; - const pathB = isIndexJSON ? basename(b.path) : b.path; - return pathA > pathB ? 1 : -1; - }); -}; - -/** - * Custom Jest sequencer for Storybook tests that ensures consistent - * test ordering and proper sharding support for distributed test execution. - */ -class StorybookTestSequencer extends Sequencer { - shard(tests: Array, { shardIndex, shardCount }: ShardOptions) { - const shardSize = Math.ceil(tests.length / shardCount); - const shardStart = shardSize * (shardIndex - 1); - const shardEnd = shardSize * shardIndex; - - return sortForIndexJSON(tests).slice(shardStart, shardEnd); - } - - sort(tests: Array) { - return sortForIndexJSON(tests); - } -} - -export default StorybookTestSequencer; diff --git a/src/test-storybook.ts b/src/test-storybook.ts index 296fde2b..be7de294 100644 --- a/src/test-storybook.ts +++ b/src/test-storybook.ts @@ -214,9 +214,7 @@ async function getIndexTempDir(url: string) { const indexJson = await getIndexJson(url); const titleIdToTest = transformPlaywrightJson(indexJson); - tmpDir = tempy.directory({ - prefix: 'test-storybook-index-json__', - }); + tmpDir = tempy.directory(); for (const [titleId, test] of Object.entries(titleIdToTest)) { const tmpFile = path.join(tmpDir, `${titleId}.test.js`); fs.writeFileSync(tmpFile, test); @@ -377,6 +375,7 @@ const main = async () => { indexTmpDir = await getIndexTempDir(targetURL); process.env.TEST_ROOT = indexTmpDir; process.env.TEST_MATCH = '**/*.test.js'; + process.env.TEST_INDEX_JSON = 'true'; } const { storiesPaths, lazyCompilation, disableTelemetry, enableCrashReports } = diff --git a/tsup.config.ts b/tsup.config.ts index 88f8f4a5..2c494909 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'tsup'; export default defineConfig([ { clean: true, - entry: ['./src/index.ts', './src/test-storybook.ts', './src/config/jest-sequencer.ts'], + entry: ['./src/index.ts', './src/test-storybook.ts', './src/config/jest-filename-sequencer.ts'], format: ['cjs', 'esm'], splitting: false, dts: true, From 66a721c991c1bd5f299b649f7ffdad8b3535c7cb Mon Sep 17 00:00:00 2001 From: matt-halliday Date: Tue, 19 Aug 2025 09:36:55 +0100 Subject: [PATCH 3/6] docs: add docblock --- src/config/jest-filename-sequencer.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/config/jest-filename-sequencer.ts b/src/config/jest-filename-sequencer.ts index 1806bc4c..ffa556f1 100644 --- a/src/config/jest-filename-sequencer.ts +++ b/src/config/jest-filename-sequencer.ts @@ -6,6 +6,11 @@ const sortByFilename = (tests: Array): Array => { return tests.sort((a, b) => basename(a.path).localeCompare(basename(b.path))); }; +/** + * Custom Jest Test Sequencer that sorts tests by their filenames. + * This ensures consistent test execution when using sharding + * against an externally deployed Storybook instance using --url. + */ class FilenameSortedTestSequencer extends Sequencer { shard(tests: Array, { shardIndex, shardCount }: ShardOptions) { const shardSize = Math.ceil(tests.length / shardCount); From ff2bfda5c5d43219702908070b3bc12db4e3afe5 Mon Sep 17 00:00:00 2001 From: matt-halliday Date: Tue, 19 Aug 2025 09:38:54 +0100 Subject: [PATCH 4/6] refactor: normal quotes --- src/config/jest-playwright.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/jest-playwright.ts b/src/config/jest-playwright.ts index c459a858..00621cc5 100644 --- a/src/config/jest-playwright.ts +++ b/src/config/jest-playwright.ts @@ -97,7 +97,7 @@ export const getJestConfig = (): Config.InitialOptions => { exitOnPageError: false, }, }, - testSequencer: TEST_INDEX_JSON ? require.resolve(`./config/jest-filename-sequencer`) : undefined, + testSequencer: TEST_INDEX_JSON ? require.resolve('./config/jest-filename-sequencer') : undefined, watchPlugins: [ require.resolve('jest-watch-typeahead/filename'), require.resolve('jest-watch-typeahead/testname'), From 31fdaeb585be42e0dc5eb9d65047fbb623641e89 Mon Sep 17 00:00:00 2001 From: matt-halliday Date: Tue, 19 Aug 2025 09:46:50 +0100 Subject: [PATCH 5/6] refactor: clean up tests --- src/config/jest-filename-sequencer.test.ts | 59 ---------------------- 1 file changed, 59 deletions(-) diff --git a/src/config/jest-filename-sequencer.test.ts b/src/config/jest-filename-sequencer.test.ts index 7c8c2be2..f994937a 100644 --- a/src/config/jest-filename-sequencer.test.ts +++ b/src/config/jest-filename-sequencer.test.ts @@ -39,65 +39,6 @@ describe('FilenameSortedTestSequencer', () => { '/path/to/story-z.test.js', ]); }); - - it('should handle tests with same basename from different directories', () => { - const tests = [ - createMockTest('/components/button/Button.test.js'), - createMockTest('/pages/home/Button.test.js'), - createMockTest('/utils/helpers/Button.test.js'), - ]; - - const result = sequencer.sort(tests); - - expect(result).toHaveLength(3); - expect(result.every(test => test.path.endsWith('Button.test.js'))).toBe(true); - }); - - it('should handle mixed file extensions', () => { - const tests = [ - createMockTest('/path/component.test.tsx'), - createMockTest('/path/component.test.js'), - createMockTest('/path/component.test.ts'), - ]; - - const result = sequencer.sort(tests); - - expect(result.map(test => test.path)).toEqual([ - '/path/component.test.js', - '/path/component.test.ts', - '/path/component.test.tsx', - ]); - }); - - it('should handle empty test array', () => { - const tests: Test[] = []; - const result = sequencer.sort(tests); - expect(result).toEqual([]); - }); - - it('should handle single test', () => { - const tests = [createMockTest('/path/to/single.test.js')]; - const result = sequencer.sort(tests); - expect(result).toEqual(tests); - }); - - it('should sort complex story filenames correctly', () => { - const tests = [ - createMockTest('/stories/Button.stories.test.js'), - createMockTest('/stories/Alert.stories.test.js'), - createMockTest('/stories/Modal.stories.test.js'), - createMockTest('/stories/Card.stories.test.js'), - ]; - - const result = sequencer.sort(tests); - - expect(result.map(test => test.path)).toEqual([ - '/stories/Alert.stories.test.js', - '/stories/Button.stories.test.js', - '/stories/Card.stories.test.js', - '/stories/Modal.stories.test.js', - ]); - }); }); describe('shard', () => { From e5441526a019835bba16627f5231c54d75f4368f Mon Sep 17 00:00:00 2001 From: matt-halliday Date: Tue, 19 Aug 2025 09:47:34 +0100 Subject: [PATCH 6/6] refactor: more cleanup --- src/config/jest-filename-sequencer.test.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/config/jest-filename-sequencer.test.ts b/src/config/jest-filename-sequencer.test.ts index f994937a..f4a543c3 100644 --- a/src/config/jest-filename-sequencer.test.ts +++ b/src/config/jest-filename-sequencer.test.ts @@ -1,7 +1,6 @@ import FilenameSortedTestSequencer from './jest-filename-sequencer'; import type { Test } from '@jest/test-result'; -// Mock test data factory const createMockTest = (path: string): Test => ({ context: { config: { @@ -52,7 +51,6 @@ describe('FilenameSortedTestSequencer', () => { createMockTest('/path/c-test.js'), ]; - // Test first shard (1 of 3) const shard1 = sequencer.shard(tests, { shardIndex: 1, shardCount: 3 }); expect(shard1.length).toBe(2); expect(shard1.map(test => test.path)).toEqual([ @@ -60,7 +58,6 @@ describe('FilenameSortedTestSequencer', () => { '/path/b-test.js', ]); - // Test second shard (2 of 3) const shard2 = sequencer.shard(tests, { shardIndex: 2, shardCount: 3 }); expect(shard2.length).toBe(2); expect(shard2.map(test => test.path)).toEqual([ @@ -68,7 +65,6 @@ describe('FilenameSortedTestSequencer', () => { '/path/d-test.js', ]); - // Test third shard (3 of 3) const shard3 = sequencer.shard(tests, { shardIndex: 3, shardCount: 3 }); expect(shard3.length).toBe(2); expect(shard3.map(test => test.path)).toEqual([