diff --git a/.storybook/preview.ts b/.storybook/preview.ts index 764e0755..ec0e3751 100644 --- a/.storybook/preview.ts +++ b/.storybook/preview.ts @@ -14,6 +14,7 @@ const preview: Preview = { decorators: [withSkippableTests], parameters: { a11y: { + test: 'error', config: { rules: [ { diff --git a/.storybook/test-runner.ts b/.storybook/test-runner.ts index d6eb5a10..01ac6d02 100644 --- a/.storybook/test-runner.ts +++ b/.storybook/test-runner.ts @@ -1,5 +1,4 @@ import { toMatchImageSnapshot } from 'jest-image-snapshot'; -import { injectAxe, checkA11y, configureAxe } from 'axe-playwright'; import { getStoryContext, waitForPageReady } from '../dist'; import type { TestRunnerConfig } from '../dist'; @@ -21,9 +20,6 @@ const config: TestRunnerConfig = { setup() { expect.extend({ toMatchImageSnapshot }); }, - async preVisit(page) { - await injectAxe(page); - }, async postVisit(page, context) { // Get entire context of a story, including parameters, args, argTypes, etc. const { parameters } = await getStoryContext(page, context); @@ -51,18 +47,6 @@ const config: TestRunnerConfig = { const innerHTML = await elementHandler?.innerHTML(); // HTML snapshot tests expect(innerHTML).toMatchSnapshot(); - - await configureAxe(page, { - rules: parameters?.a11y?.config?.rules, - }); - - const element = parameters?.a11y?.element ?? 'body'; - await checkA11y(page, element, { - detailedReport: true, - detailedReportOptions: { - html: true, - }, - }); }, }; diff --git a/README.md b/README.md index 6f40118a..a7ca95fd 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,8 @@ Storybook test runner turns all of your stories into executable tests. - [Recipes](#recipes) - [Preconfiguring viewport size](#preconfiguring-viewport-size) - [Accessibility testing](#accessibility-testing) + - [With Storybook 9](#with-storybook-9) + - [With Storybook 8](#with-storybook-8) - [DOM snapshot (HTML)](#dom-snapshot-html) - [Image snapshot](#image-snapshot) - [Troubleshooting](#troubleshooting) @@ -167,7 +169,7 @@ Usage: test-storybook [options] | `--json` | Prints the test results in JSON. This mode will send all other test output and user messages to stderr.
`test-storybook --json` | | `--outputFile` | Write test results to a file when the --json option is also specified.
`test-storybook --json --outputFile results.json` | | `--junit` | Indicates that test information should be reported in a junit file.
`test-storybook --**junit**` | -| `--listTests` | Lists all test files that will be run, and exits
`test-storybook --listTests` | +| `--listTests` | Lists all test files that will be run, and exits
`test-storybook --listTests` | | `--ci` | Instead of the regular behavior of storing a new snapshot automatically, it will fail the test and require Jest to be run with `--updateSnapshot`.
`test-storybook --ci` | | `--shard [shardIndex/shardCount]` | Splits your test suite across different machines to run in CI.
`test-storybook --shard=1/3` | | `--failOnConsole` | Makes tests fail on browser console errors
`test-storybook --failOnConsole` | @@ -885,6 +887,32 @@ export default config; ### Accessibility testing +#### With Storybook 9 + +In Storybook 9, the accessibility addon has been enhanced with automated reporting capabilities and the Test-runner has out of the box support for it. If you have `@storybook/addon-a11y` installed, as long as you enable them via parameters, you will get a11y checks for every story: + +```ts +// .storybook/preview.ts + +const preview = { + parameters: { + a11y: { + // 'error' will cause a11y violations to fail tests + test: 'error', // or 'todo' or 'off' + }, + }, +}; + +export default preview; +``` + +If you had a11y tests set up previously for Storybook 8 (with the recipe below), you can uninstall `axe-playwright` and remove all the code from the test-runner hooks, as they are not necessary anymore. + +#### With Storybook 8 + +> [!TIP] +> If you upgrade to Storybook 9, there is out of the box support for a11y tests and you don't have to follow a recipe like this. + You can install `axe-playwright` and use it in tandem with the test-runner to test the accessibility of your components. If you use [`@storybook/addon-a11y`](https://storybook.js.org/addons/@storybook/addon-a11y), you can reuse its parameters and make sure that the tests match in configuration, both in the accessibility addon panel and the test-runner. diff --git a/package.json b/package.json index c34bd62f..553e7444 100644 --- a/package.json +++ b/package.json @@ -85,7 +85,6 @@ "@types/node-fetch": "^2.6.11", "@vitejs/plugin-react": "^4.0.3", "auto": "^11.1.6", - "axe-playwright": "^2.1.0", "babel-jest": "^29.0.0", "babel-loader": "^8.1.0", "babel-plugin-istanbul": "^6.1.1", diff --git a/src/setup-page-script.ts b/src/setup-page-script.ts index 440934c9..51e2d0e4 100644 --- a/src/setup-page-script.ts +++ b/src/setup-page-script.ts @@ -4,6 +4,8 @@ * This file is a template to the content which is injected to the Playwright page via the ./setup-page.ts file. * setup-page.ts will read the contents of this file and replace values that use {{x}} pattern, and they should be put right below: */ +import { PreviewWeb } from 'storybook/internal/preview-api'; +import { StoryContext } from 'storybook/internal/csf'; type ConsoleMethod = | 'log' @@ -32,6 +34,7 @@ declare global { // this is defined in setup-page.ts and can be used for logging from the browser to node, helpful for debugging var logToPage: (message: string) => Promise; var testRunner_errorMessageFormatter: (message: string) => Promise; + var __STORYBOOK_PREVIEW__: PreviewWeb; } // Type definitions for function parameters and return types @@ -42,6 +45,7 @@ const magenta: Colorizer = (message: string) => `\u001b[35m${message}\u001b[39m` const blue: Colorizer = (message: string) => `\u001b[34m${message}\u001b[39m`; const red: Colorizer = (message: string) => `\u001b[31m${message}\u001b[39m`; const yellow: Colorizer = (message: string) => `\u001b[33m${message}\u001b[39m`; +const grey: Colorizer = (message: string) => `\u001b[90m${message}\u001b[39m`; // Constants var LIMIT_REPLACE_NODE = '[...]'; @@ -204,36 +208,53 @@ function addToUserAgent(extra: string): void { } } +function getStory(): StoryContext { + const currentRender = globalThis.__STORYBOOK_PREVIEW__.currentRender; + if (currentRender && 'story' in currentRender) { + return currentRender.story as unknown as StoryContext; + } + + return {} as StoryContext; +} + // Custom error class class StorybookTestRunnerError extends Error { - constructor( - storyId: string, - errorMessage: string, - logs: string[] = [], - isMessageFormatted: boolean = false - ) { + constructor(params: { + storyId: string; + errorMessage: string; + logs?: string[]; + isMessageFormatted?: boolean; + }) { + const { storyId, errorMessage, logs = [], isMessageFormatted = false } = params; const message = isMessageFormatted ? errorMessage - : StorybookTestRunnerError.buildErrorMessage(storyId, errorMessage, logs); + : StorybookTestRunnerError.buildErrorMessage({ storyId, errorMessage, logs }); super(message); this.name = 'StorybookTestRunnerError'; } - public static buildErrorMessage( - storyId: string, - errorMessage: string, - logs: string[] = [] - ): string { + public static buildErrorMessage(params: { + storyId: string; + errorMessage: string; + logs?: string[]; + panel?: string; + errorMessagePrefix?: string; + }): string { + const { storyId, errorMessage, logs = [], panel, errorMessagePrefix = '' } = params; const storyUrl = `${TEST_RUNNER_STORYBOOK_URL}?path=/story/${storyId}`; - const finalStoryUrl = `${storyUrl}&addonPanel=storybook/interactions/panel`; + const finalStoryUrl = panel ? `${storyUrl}&addonPanel=${panel}` : storyUrl; const separator = '\n\n--------------------------------------------------'; // The original error message will also be collected in the logs, so we filter it to avoid duplication const finalLogs = logs.filter((err: string) => !err.includes(errorMessage)); const extraLogs = finalLogs.length > 0 ? separator + '\n\nBrowser logs:\n\n' + finalLogs.join('\n\n') : ''; - const message = `\nAn error occurred in the following story. Access the link for full output:\n${finalStoryUrl}\n\nMessage:\n ${truncate( + const linkPrefix = blue( + `\nClick to debug the error directly in Storybook:\n${finalStoryUrl}\n\n` + ); + + const message = `${errorMessagePrefix}${linkPrefix}Message:\n ${truncate( errorMessage, TEST_RUNNER_DEBUG_PRINT_LIMIT )}\n${extraLogs}`; @@ -244,7 +265,7 @@ class StorybookTestRunnerError extends Error { // @ts-expect-error Global function to throw custom error, used by the test runner or user async function __throwError(storyId: string, errorMessage: string, logs: string[]): Promise { - throw new StorybookTestRunnerError(storyId, errorMessage, logs); + throw new StorybookTestRunnerError({ storyId, errorMessage, logs }); } // Wait for Storybook to load @@ -277,7 +298,6 @@ async function __waitForStorybook(): Promise { // Get context from Storybook // @ts-expect-error Global function to get context, used by the test runner or user async function __getContext(storyId: string): Promise { - // @ts-expect-error globally defined via Storybook return globalThis.__STORYBOOK_PREVIEW__.storyStore.loadStory({ storyId }); } @@ -296,16 +316,17 @@ async function __test(storyId: string): Promise { await __waitForStorybook(); } catch (err) { const message = `Timed out waiting for Storybook to load after 10 seconds. Are you sure the Storybook is running correctly in that URL? Is the Storybook private (e.g. under authentication layers)?\n\n\nHTML: ${document.body.innerHTML}`; - throw new StorybookTestRunnerError(storyId, message); + throw new StorybookTestRunnerError({ storyId, errorMessage: message }); } // @ts-expect-error globally defined via Storybook const channel = globalThis.__STORYBOOK_ADDONS_CHANNEL__; if (!channel) { - throw new StorybookTestRunnerError( + throw new StorybookTestRunnerError({ storyId, - 'The test runner could not access the Storybook channel. Are you sure the Storybook is running correctly in that URL?' - ); + errorMessage: + 'The test runner could not access the Storybook channel. Are you sure the Storybook is running correctly in that URL?', + }); } addToUserAgent(`(StorybookTestRunner@${TEST_RUNNER_VERSION})`); @@ -385,32 +406,73 @@ async function __test(storyId: string): Promise { }; return new Promise((resolve, reject) => { - const rejectWithFormattedError = (storyId: string, message: string) => { - const errorMessage = StorybookTestRunnerError.buildErrorMessage(storyId, message, logs); + const rejectWithFormattedError = (storyId: string, message: string, panel?: string) => { + const errorMessage = StorybookTestRunnerError.buildErrorMessage({ + storyId, + errorMessage: message, + logs, + panel, + }); testRunner_errorMessageFormatter(errorMessage) .then((formattedMessage) => { - reject(new StorybookTestRunnerError(storyId, formattedMessage, logs, true)); + reject( + new StorybookTestRunnerError({ + storyId, + errorMessage: formattedMessage, + logs, + isMessageFormatted: true, + }) + ); }) .catch((error) => { reject( - new StorybookTestRunnerError( + new StorybookTestRunnerError({ storyId, - 'There was an error when executing the errorMessageFormatter defiend in your Storybook test-runner config file. Please fix it and rerun the tests:\n\n' + - error.message - ) + errorMessage: + 'There was an error when executing the errorMessageFormatter defiend in your Storybook test-runner config file. Please fix it and rerun the tests:\n\n' + + error.message, + }) ); }); }; + const INTERACTIONS_PANEL = 'storybook/interactions/panel'; + const A11Y_PANEL = 'storybook/a11y/panel'; + const listeners = { - [TEST_RUNNER_RENDERED_EVENT]: () => { + [TEST_RUNNER_RENDERED_EVENT]: (data: any) => { cleanup(listeners); + if (hasErrors) { rejectWithFormattedError(storyId, 'Browser console errors'); - } else { - resolve(document.getElementById('root')); + return; + } else if (data?.reporters) { + const story = getStory(); + const a11yTestParameter = story?.parameters?.a11y?.test; + const a11yReport = data.reporters.find((reporter: any) => reporter.type === 'a11y'); + if ( + a11yReport.result?.violations?.length > 0 && + (a11yTestParameter === 'error' || a11yTestParameter === 'todo') + ) { + const violations = expectToHaveNoViolations(a11yReport.result); + if (violations && a11yTestParameter === 'error') { + rejectWithFormattedError(storyId, violations.long, A11Y_PANEL); + return; + } else if (violations && a11yTestParameter === 'todo') { + const warningMessage = StorybookTestRunnerError.buildErrorMessage({ + storyId, + errorMessagePrefix: `--------------------------\n${story.title} > ${story.name}`, + errorMessage: yellow(violations.short), + logs, + panel: A11Y_PANEL, + }); + logToPage(warningMessage); + } + } } + + resolve(document.getElementById('root')); }, storyUnchanged: () => { @@ -420,23 +482,23 @@ async function __test(storyId: string): Promise { storyErrored: ({ description }: { description: string }) => { cleanup(listeners); - rejectWithFormattedError(storyId, description); + rejectWithFormattedError(storyId, description, INTERACTIONS_PANEL); }, storyThrewException: (error: Error) => { cleanup(listeners); - rejectWithFormattedError(storyId, error.message); + rejectWithFormattedError(storyId, error.message, INTERACTIONS_PANEL); }, playFunctionThrewException: (error: Error) => { cleanup(listeners); - rejectWithFormattedError(storyId, error.message); + rejectWithFormattedError(storyId, error.message, INTERACTIONS_PANEL); }, unhandledErrorsWhilePlaying: ([error]: Error[]) => { cleanup(listeners); - rejectWithFormattedError(storyId, error.message); + rejectWithFormattedError(storyId, error.message, INTERACTIONS_PANEL); }, storyMissing: (id: string) => { @@ -455,4 +517,65 @@ async function __test(storyId: string): Promise { }); } +function expectToHaveNoViolations(results: any): { long: string; short: string } | null { + let violations = filterViolations( + results.violations, + // `impactLevels` is not a valid toolOption but one we add to the config + // when calling `run`. axe just happens to pass this along. Might be a safer + // way to do this since it's not documented API. + results.toolOptions?.impactLevels ?? [] + ); + + function reporter(violations: any) { + if (violations.length === 0) { + return null; + } + + let lineBreak = '\n\n'; + let horizontalLine = '\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500'; + + return violations + .map((violation: any) => { + let errorBody = violation.nodes + .map((node: any) => { + let selector = node.target.join(', '); + let expectedText = + red(`Expected the HTML found at $('${selector}') to have no violations:`) + lineBreak; + return ( + expectedText + + grey(node.html) + + lineBreak + + red(`Received:`) + + lineBreak + + red(`"${violation.help} (${violation.id})"`) + + lineBreak + + yellow(node.failureSummary) + + lineBreak + + (violation.helpUrl + ? red(`You can find more information on this issue here:`) + + `\n${blue(violation.helpUrl)}` + : '') + ); + }) + .join(lineBreak); + return errorBody; + }) + .join(lineBreak + horizontalLine + lineBreak); + } + + let formatedViolations = reporter(violations); + + return { + long: formatedViolations, + short: `Found ${violations.length} a11y violations, run the test with 'a11y: { test: 'error' }' parameter to see the full report or debug it directly in Storybook.`, + }; +} + +function filterViolations(violations: any, impactLevels: Array) { + if (impactLevels && impactLevels.length > 0) { + return violations.filter((v: any) => impactLevels.includes(v.impact)); + } + return violations; +} + export {}; diff --git a/yarn.lock b/yarn.lock index b985ba5f..34a428b0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2951,7 +2951,6 @@ __metadata: "@types/node-fetch": "npm:^2.6.11" "@vitejs/plugin-react": "npm:^4.0.3" auto: "npm:^11.1.6" - axe-playwright: "npm:^2.1.0" babel-jest: "npm:^29.0.0" babel-loader: "npm:^8.1.0" babel-plugin-istanbul: "npm:^6.1.1" @@ -3340,13 +3339,6 @@ __metadata: languageName: node linkType: hard -"@types/junit-report-builder@npm:^3.0.2": - version: 3.0.2 - resolution: "@types/junit-report-builder@npm:3.0.2" - checksum: 10/7fead0b771f95cd8e607223ace2f43cc881b3e7944db405f903590b56379d14132032b7d89c9afa7dff266b95f516a476a39ad40f9200bfb61fc8ed7f6b1bff6 - languageName: node - linkType: hard - "@types/mdx@npm:^2.0.0": version: 2.0.13 resolution: "@types/mdx@npm:2.0.13" @@ -3820,39 +3812,13 @@ __metadata: languageName: node linkType: hard -"axe-core@npm:^4.10.1, axe-core@npm:^4.2.0": +"axe-core@npm:^4.2.0": version: 4.10.3 resolution: "axe-core@npm:4.10.3" checksum: 10/9ff51ad0fd0fdec5c0247ea74e8ace5990b54c7f01f8fa3e5cd8ba98b0db24d8ebd7bab4a9bd4d75c28c4edcd1eac455b44c8c6c258c6a98f3d2f88bc60af4cc languageName: node linkType: hard -"axe-html-reporter@npm:2.2.11": - version: 2.2.11 - resolution: "axe-html-reporter@npm:2.2.11" - dependencies: - mustache: "npm:^4.0.1" - peerDependencies: - axe-core: ">=3" - checksum: 10/489a904c62fe5c74db9490773a624e8573906b54a9c8264b831ee50b648f623ba292a011598d257bd6b8dc35eede21998fc25c799bd99eb9ca8ecf2d05ef92a1 - languageName: node - linkType: hard - -"axe-playwright@npm:^2.1.0": - version: 2.1.0 - resolution: "axe-playwright@npm:2.1.0" - dependencies: - "@types/junit-report-builder": "npm:^3.0.2" - axe-core: "npm:^4.10.1" - axe-html-reporter: "npm:2.2.11" - junit-report-builder: "npm:^5.1.1" - picocolors: "npm:^1.1.1" - peerDependencies: - playwright: ">1.0.0" - checksum: 10/d478c206bdd950257fafc872a6da3d9e73ca6a235ecea504bfde9744ca9265cc7abea050301c7cb611a3e79d61e14d50a047595c6e6cc13353e6c1c835afe922 - languageName: node - linkType: hard - "axios@npm:^1.6.1": version: 1.6.2 resolution: "axios@npm:1.6.2" @@ -6973,17 +6939,6 @@ __metadata: languageName: node linkType: hard -"junit-report-builder@npm:^5.1.1": - version: 5.1.1 - resolution: "junit-report-builder@npm:5.1.1" - dependencies: - lodash: "npm:^4.17.21" - make-dir: "npm:^3.1.0" - xmlbuilder: "npm:^15.1.1" - checksum: 10/273678301654c22265a8ccd5a605f498747e2530e801b332f1033fa3e35b3fc3538e10b335674cca41bbc1f4c9476350254820e1f5f0cf4101ac8dd4c1402bf8 - languageName: node - linkType: hard - "kleur@npm:^3.0.3": version: 3.0.3 resolution: "kleur@npm:3.0.3" @@ -7554,15 +7509,6 @@ __metadata: languageName: node linkType: hard -"mustache@npm:^4.0.1": - version: 4.2.0 - resolution: "mustache@npm:4.2.0" - bin: - mustache: bin/mustache - checksum: 10/6e668bd5803255ab0779c3983b9412b5c4f4f90e822230e0e8f414f5449ed7a137eed29430e835aa689886f663385cfe05f808eb34b16e1f3a95525889b05cd3 - languageName: node - linkType: hard - "mz@npm:^2.7.0": version: 2.7.0 resolution: "mz@npm:2.7.0" @@ -10365,13 +10311,6 @@ __metadata: languageName: node linkType: hard -"xmlbuilder@npm:^15.1.1": - version: 15.1.1 - resolution: "xmlbuilder@npm:15.1.1" - checksum: 10/e6f4bab2504afdd5f80491bda948894d2146756532521dbe7db33ae0931cd3000e3b4da19b3f5b3f51bedbd9ee06582144d28136d68bd1df96579ecf4d4404a2 - languageName: node - linkType: hard - "y18n@npm:^4.0.0": version: 4.0.3 resolution: "y18n@npm:4.0.3"