diff --git a/README.md b/README.md index 8a80fac30..92ec97b8a 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ [![Stand With Ukraine](https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner-direct-single.svg)](https://stand-with-ukraine.pp.ua) [](https://join.slack.com/t/codeceptjs/shared_invite/enQtMzA5OTM4NDM2MzA4LWE4MThhN2NmYTgxNTU5MTc4YzAyYWMwY2JkMmZlYWI5MWQ2MDM5MmRmYzZmYmNiNmY5NTAzM2EwMGIwOTNhOGQ) [](https://codecept.discourse.group) [![NPM version][npm-image]][npm-url] [](https://hub.docker.com/r/codeceptjs/codeceptjs) -[![AI features](https://img.shields.io/badge/AI-features?logo=openai&logoColor=white)](https://github.com/codeceptjs/CodeceptJS/edit/3.x/docs/ai.md) [![StandWithUkraine](https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/badges/StandWithUkraine.svg)](https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md) +[![AI features](https://img.shields.io/badge/AI-features?logo=openai&logoColor=white)](https://github.com/codeceptjs/CodeceptJS/edit/3.x/docs/ai.md) [![MCP Server](https://img.shields.io/badge/MCP-server?logo=anthropic&logoColor=white)](https://github.com/codeceptjs/CodeceptJS/blob/main/docs/mcp.md) [![StandWithUkraine](https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/badges/StandWithUkraine.svg)](https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md) ## Build Status @@ -276,6 +276,84 @@ Full support for Gherkin scenarios with proper feature formatting: The HTML reporter generates self-contained reports that can be easily shared with your team. Learn more about configuration and features in the [HTML Reporter documentation](https://codecept.io/plugins/#htmlreporter). +## MCP Server + +CodeceptJS includes a [Model Context Protocol (MCP)](https://modelcontextprotocol.io) server that enables AI agents (like Claude, ChatGPT) to interact with and control your tests programmatically. This allows AI to: + +- **List all tests** in your project +- **List all available actions** (I.* methods) from enabled helpers +- **Run arbitrary CodeceptJS code** and capture artifacts (ARIA snapshots, screenshots, HTML, logs) +- **Run specific tests** with AI-friendly trace files generated by the aiTrace plugin +- **Run tests step by step** for detailed debugging and analysis +- **Control browser sessions** (start/stop) + +### Features + +- **AI-Friendly Trace Files**: When enabled, the aiTrace plugin generates comprehensive trace files with screenshots, HTML, ARIA snapshots, browser logs, and HTTP requests +- **Real-time Monitoring**: Get trace file URLs at the start of test execution so AI can monitor tests in real-time +- **Artifact Capture**: Automatically capture ARIA snapshots, page source, console logs, and more +- **Browser Management**: Automatic or manual browser lifecycle control +- **Timeout Control**: Configurable timeouts for all operations + +### Quick Start + +1. Install the MCP SDK: +```sh +npm install @modelcontextprotocol/sdk +``` + +2. Configure your MCP client (e.g., Claude Desktop): +```json +{ + "mcpServers": { + "codeceptjs": { + "command": "node", + "args": ["path/to/codeceptjs/bin/mcp-server.js"] + } + } +} +``` + +3. Enable the aiTrace plugin in `codecept.conf.js`: +```javascript +plugins: { + aiTrace: { + enabled: true + } +} +``` + +### Usage Examples + +**List all tests:** +```json +{ + "name": "list_tests" +} +``` + +**Run CodeceptJS code:** +```json +{ + "name": "run_code", + "arguments": { + "code": "await I.amOnPage('/'); await I.see('Welcome')" + } +} +``` + +**Run a test with trace:** +```json +{ + "name": "run_test", + "arguments": { + "test": "tests/login_test.js" + } +} +``` + +Learn more about the MCP Server in the [MCP documentation](https://github.com/codeceptjs/CodeceptJS/blob/main/docs/mcp.md). + ## PageObjects CodeceptJS provides the most simple way to create and use page objects in your test. diff --git a/bin/mcp-server.js b/bin/mcp-server.js new file mode 100644 index 000000000..e03856234 --- /dev/null +++ b/bin/mcp-server.js @@ -0,0 +1,610 @@ +import { Server } from '@modelcontextprotocol/sdk/server/index.js' +import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js' +import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js' +import Codecept from '../lib/codecept.js' +import container from '../lib/container.js' +import { getParamsToString } from '../lib/parser.js' +import { methodsOfObject } from '../lib/utils.js' +import event from '../lib/event.js' +import { fileURLToPath } from 'url' +import { dirname, resolve as resolvePath } from 'path' +import path from 'path' +import crypto from 'crypto' +import { spawn } from 'child_process' +import { createRequire } from 'module' +import { existsSync, readdirSync } from 'fs' + +const require = createRequire(import.meta.url) + +const __filename = fileURLToPath(import.meta.url) +const __dirname = dirname(__filename) + +let codecept = null +let containerInitialized = false +let browserStarted = false + +let runLock = Promise.resolve() +async function withLock(fn) { + const prev = runLock + let release + runLock = new Promise(r => (release = r)) + await prev + try { return await fn() } + finally { release() } +} + +async function withSilencedIO(fn) { + const origOut = process.stdout.write.bind(process.stdout) + const origErr = process.stderr.write.bind(process.stderr) + + process.stdout.write = () => true + process.stderr.write = () => true + + try { + return await fn() + } finally { + process.stdout.write = origOut + process.stderr.write = origErr + } +} + +function runCmd(cmd, args, { cwd = process.cwd(), timeout = 60000 } = {}) { + return new Promise((resolve, reject) => { + const child = spawn(cmd, args, { + cwd, + env: { ...process.env, NODE_ENV: process.env.NODE_ENV || 'test' }, + stdio: ['ignore', 'pipe', 'pipe'], + }) + + let out = '' + let err = '' + + const t = setTimeout(() => { + child.kill('SIGKILL') + reject(new Error(`Timeout after ${timeout}ms`)) + }, timeout) + + child.stdout.on('data', d => (out += d.toString('utf8'))) + child.stderr.on('data', d => (err += d.toString('utf8'))) + + child.on('error', e => { + clearTimeout(t) + reject(e) + }) + + child.on('close', code => { + clearTimeout(t) + resolve({ code, out, err }) + }) + }) +} + +function resolveConfigPath(configPath) { + const cwd = process.cwd() + const envRoot = process.env.CODECEPTJS_PROJECT_DIR + + if (configPath && !path.isAbsolute(configPath)) { + const base = envRoot || cwd + configPath = path.resolve(base, configPath) + } + + if (!configPath) { + const base = envRoot || cwd + configPath = process.env.CODECEPTJS_CONFIG || path.resolve(base, 'codecept.conf.js') + if (!existsSync(configPath)) configPath = path.resolve(base, 'codecept.conf.cjs') + } + + if (!existsSync(configPath)) { + throw new Error( + `CodeceptJS config not found: ${configPath}\n` + + `CODECEPTJS_CONFIG=${process.env.CODECEPTJS_CONFIG || 'not set'}\n` + + `CODECEPTJS_PROJECT_DIR=${process.env.CODECEPTJS_PROJECT_DIR || 'not set'}\n` + + `cwd=${cwd}` + ) + } + + return { configPath, configDir: path.dirname(configPath) } +} + +function findCodeceptCliUpwards(startDir, { maxUp = 8 } = {}) { + let dir = startDir + + for (let i = 0; i <= maxUp; i++) { + const candidates = [ + path.resolve(dir, 'bin', 'codecept.js'), + path.resolve(dir, 'node_modules', 'codeceptjs', 'bin', 'codecept.js'), + path.resolve(dir, 'node_modules', '.bin', 'codeceptjs.cmd'), + path.resolve(dir, 'node_modules', '.bin', 'codeceptjs'), + ] + + for (const p of candidates) { + if (existsSync(p)) return { cli: p, root: dir } + } + + try { + const pkgJson = require.resolve('codeceptjs/package.json', { paths: [dir] }) + const pkgDir = path.dirname(pkgJson) + const jsCli = path.resolve(pkgDir, 'bin', 'codecept.js') + if (existsSync(jsCli)) return { cli: jsCli, root: dir } + } catch {} + + const parent = path.dirname(dir) + if (parent === dir) break + dir = parent + } + + throw new Error(`Cannot find CodeceptJS CLI walking up from: ${startDir}`) +} + +function looksLikePath(v) { + return typeof v === 'string' && ( + v.includes('/') || v.includes('\\') || + v.endsWith('.js') || v.endsWith('.ts') + ) +} + +function normalizePath(p) { + return String(p).replace(/\\/g, '/') +} + +function findFileByBasename(rootDir, baseNames, { maxDepth = 8 } = {}) { + const targets = new Set(baseNames.map(x => x.toLowerCase())) + + function walk(dir, depth) { + if (depth > maxDepth) return null + + let entries + try { entries = readdirSync(dir, { withFileTypes: true }) } catch { return null } + + for (const e of entries) { + const full = path.join(dir, e.name) + + if (e.isDirectory()) { + if (e.name === 'node_modules' || e.name === '.git' || e.name === 'output') continue + const res = walk(full, depth + 1) + if (res) return res + continue + } + + if (targets.has(e.name.toLowerCase())) return full + } + + return null + } + + return walk(rootDir, 0) +} + +async function listTestsJson({ cli, root, configPath }) { + const args = ['list', '--config', configPath, '--json'] + const isNodeScript = cli.endsWith('.js') + + const res = isNodeScript + ? await runCmd(process.execPath, [cli, ...args], { cwd: root, timeout: 60000 }) + : await runCmd(cli, args, { cwd: root, timeout: 60000 }) + + const out = (res.out || '').trim() + try { return JSON.parse(out) } catch { return null } +} + +function extractFilesFromListJson(json) { + if (!json) return [] + if (Array.isArray(json)) return json.map(String) + if (Array.isArray(json.tests)) return json.tests.map(String) + if (Array.isArray(json.files)) return json.files.map(String) + if (Array.isArray(json.testFiles)) return json.testFiles.map(String) + return [] +} + +async function resolveTestToFile({ cli, root, configPath, test }) { + if (looksLikePath(test)) return test + + const raw = String(test).trim() + const candidates = [ + raw, + `${raw}.js`, + `${raw}.ts`, + `${raw}_test.js`, + `${raw}.test.js`, + ].map(x => x.toLowerCase()) + + const json = await listTestsJson({ cli, root, configPath }) + const files = extractFilesFromListJson(json).map(normalizePath) + + if (files.length) { + const byName = files.find(f => candidates.some(c => path.basename(f).toLowerCase() === c)) + if (byName) return byName + + const byContains = files.find(f => f.toLowerCase().includes(raw.toLowerCase())) + if (byContains) return byContains + } + + const fsFound = findFileByBasename(root, candidates) + return fsFound ? normalizePath(fsFound) : null +} + +function clearString(str) { + return str.replace(/[^a-zA-Z0-9]/g, '_') +} + +function getTraceDir(testTitle, testFile) { + const hash = crypto.createHash('sha256').update(testFile + testTitle).digest('hex').slice(0, 8) + const cleanTitle = clearString(testTitle).slice(0, 200) + const outputDir = global.output_dir || resolvePath(process.cwd(), 'output') + return resolvePath(outputDir, `trace_${cleanTitle}_${hash}`) +} + +async function initCodecept(configPath) { + if (containerInitialized) return + + const testRoot = process.env.CODECEPTJS_PROJECT_DIR || process.cwd() + + if (!configPath) { + configPath = process.env.CODECEPTJS_CONFIG || resolvePath(testRoot, 'codecept.conf.js') + if (!existsSync(configPath)) configPath = resolvePath(testRoot, 'codecept.conf.cjs') + } + + if (!existsSync(configPath)) { + throw new Error( + `CodeceptJS config not found: ${configPath}\n` + + `CODECEPTJS_CONFIG=${process.env.CODECEPTJS_CONFIG || 'not set'}\n` + + `CODECEPTJS_PROJECT_DIR=${process.env.CODECEPTJS_PROJECT_DIR || 'not set'}\n` + + `cwd=${process.cwd()}` + ) + } + + console.log = () => {} + console.error = () => {} + console.warn = () => {} + + const { getConfig } = await import('../lib/command/utils.js') + const config = await getConfig(configPath) + + codecept = new Codecept(config, {}) + await codecept.init(testRoot) + await container.create(config, {}) + await container.started() + + containerInitialized = true + browserStarted = true +} + +const server = new Server( + { name: 'codeceptjs-mcp-server', version: '1.0.0' }, + { capabilities: { tools: {} } } +) + +server.setRequestHandler(ListToolsRequestSchema, async () => ({ + tools: [ + { + name: 'list_tests', + description: 'List all tests in the CodeceptJS project', + inputSchema: { type: 'object', properties: { config: { type: 'string' } } }, + }, + { + name: 'list_actions', + description: 'List all available CodeceptJS actions (I.* methods)', + inputSchema: { type: 'object', properties: { config: { type: 'string' } } }, + }, + { + name: 'run_code', + description: 'Run arbitrary CodeceptJS code.', + inputSchema: { + type: 'object', + properties: { + code: { type: 'string' }, + timeout: { type: 'number' }, + config: { type: 'string' }, + saveArtifacts: { type: 'boolean' }, + }, + required: ['code'], + }, + }, + { + name: 'run_test', + description: 'Run a specific test.', + inputSchema: { + type: 'object', + properties: { + test: { type: 'string' }, + timeout: { type: 'number' }, + config: { type: 'string' }, + }, + required: ['test'], + }, + }, + { + name: 'run_step_by_step', + description: 'Run a test step by step with pauses between steps.', + inputSchema: { + type: 'object', + properties: { + test: { type: 'string' }, + timeout: { type: 'number' }, + config: { type: 'string' }, + }, + required: ['test'], + }, + }, + { + name: 'start_browser', + description: 'Start the browser session.', + inputSchema: { type: 'object', properties: { config: { type: 'string' } } }, + }, + { + name: 'stop_browser', + description: 'Stop the browser session.', + inputSchema: { type: 'object', properties: {} }, + }, + ], +})) + +server.setRequestHandler(CallToolRequestSchema, async (request) => { + const { name, arguments: args } = request.params + + try { + switch (name) { + case 'list_tests': { + const configPath = args?.config + await initCodecept(configPath) + + codecept.loadTests() + const tests = codecept.testFiles.map(testFile => { + const relativePath = testFile.replace(process.cwd(), '').replace(/\\/g, '/') + return { + file: testFile, + relativePath: relativePath.startsWith('/') ? relativePath.slice(1) : relativePath, + } + }) + + return { content: [{ type: 'text', text: JSON.stringify({ count: tests.length, tests }, null, 2) }] } + } + + case 'list_actions': { + const configPath = args?.config + await initCodecept(configPath) + + const helpers = container.helpers() + const supportI = container.support('I') + const actions = [] + const actionDetails = [] + + for (const helperName in helpers) { + const helper = helpers[helperName] + methodsOfObject(helper).forEach(action => { + if (actions.includes(action)) return + actions.push(action) + const params = getParamsToString(helper[action]) + actionDetails.push({ helper: helperName, action, signature: `I.${action}(${params})` }) + }) + } + + for (const n in supportI) { + if (actions.includes(n)) continue + const actor = supportI[n] + const params = getParamsToString(actor) + actionDetails.push({ helper: 'SupportObject', action: n, signature: `I.${n}(${params})` }) + } + + return { content: [{ type: 'text', text: JSON.stringify({ count: actionDetails.length, actions: actionDetails }, null, 2) }] } + } + + case 'start_browser': { + const configPath = args?.config + if (browserStarted) { + return { content: [{ type: 'text', text: JSON.stringify({ status: 'Browser already started' }, null, 2) }] } + } + await initCodecept(configPath) + return { content: [{ type: 'text', text: JSON.stringify({ status: 'Browser started successfully' }, null, 2) }] } + } + + case 'stop_browser': { + if (!containerInitialized) { + return { content: [{ type: 'text', text: JSON.stringify({ status: 'Browser not initialized' }, null, 2) }] } + } + + const helpers = container.helpers() + for (const helperName in helpers) { + const helper = helpers[helperName] + try { if (helper._finish) await helper._finish() } catch {} + } + + browserStarted = false + containerInitialized = false + + return { content: [{ type: 'text', text: JSON.stringify({ status: 'Browser stopped successfully' }, null, 2) }] } + } + + case 'run_code': { + const { code, timeout = 60000, config: configPath, saveArtifacts = true } = args + await initCodecept(configPath) + + const I = container.support('I') + if (!I) throw new Error('I object not available. Make sure helpers are configured.') + + const result = { status: 'unknown', output: '', error: null, artifacts: {} } + + try { + const asyncFn = new Function('I', `return (async () => { ${code} })()`) + await Promise.race([ + asyncFn(I), + new Promise((_, reject) => setTimeout(() => reject(new Error(`Timeout after ${timeout}ms`)), timeout)), + ]) + + result.status = 'success' + result.output = 'Code executed successfully' + + if (saveArtifacts) { + const helpers = container.helpers() + const helper = Object.values(helpers)[0] + if (helper) { + try { + if (helper.grabAriaSnapshot) result.artifacts.aria = await helper.grabAriaSnapshot() + if (helper.grabCurrentUrl) result.artifacts.url = await helper.grabCurrentUrl() + if (helper.grabBrowserLogs) result.artifacts.consoleLogs = (await helper.grabBrowserLogs()) || [] + if (helper.grabSource) { + const html = await helper.grabSource() + result.artifacts.html = html.substring(0, 10000) + '...' + } + } catch (e) { + result.output += ` (Warning: ${e.message})` + } + } + } + } catch (error) { + result.status = 'failed' + result.error = error.message + result.output = error.stack || error.message + } + + return { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }] } + } + + case 'run_test': { + return await withLock(async () => { + const { test, timeout = 60000, config: configPathArg } = args || {} + const { configPath, configDir } = resolveConfigPath(configPathArg) + + const { cli, root } = findCodeceptCliUpwards(configDir) + const isNodeScript = cli.endsWith('.js') + + const resolvedFile = await resolveTestToFile({ cli, root, configPath, test }) + const runArgs = ['run', '--config', configPath, '--reporter', 'json'] + + if (resolvedFile) runArgs.push(resolvedFile) + else if (looksLikePath(test)) runArgs.push(test) + else runArgs.push('--grep', String(test)) + + const res = isNodeScript + ? await runCmd(process.execPath, [cli, ...runArgs], { cwd: root, timeout }) + : await runCmd(cli, runArgs, { cwd: root, timeout }) + + const { code, out, err } = res + + let parsed = null + const jsonStart = out.indexOf('{') + const jsonEnd = out.lastIndexOf('}') + if (jsonStart !== -1 && jsonEnd !== -1 && jsonEnd > jsonStart) { + try { parsed = JSON.parse(out.slice(jsonStart, jsonEnd + 1)) } catch {} + } + + return { + content: [{ + type: 'text', + text: JSON.stringify({ + meta: { exitCode: code, cli, root, configPath, args: runArgs, resolvedFile: resolvedFile || null }, + reporterJson: parsed, + stderr: err ? err.slice(0, 20000) : '', + rawStdout: parsed ? '' : out.slice(0, 20000), + }, null, 2), + }], + } + }) + } + + case 'run_step_by_step': { + const { test, timeout = 60000, config: configPath } = args + await initCodecept(configPath) + + return await withSilencedIO(async () => { + codecept.loadTests() + + let testFiles = codecept.testFiles + if (test) { + const testName = normalizePath(test).toLowerCase() + testFiles = codecept.testFiles.filter(f => { + const filePath = normalizePath(f).toLowerCase() + return filePath.includes(testName) || filePath.endsWith(testName) + }) + } + + if (!testFiles.length) throw new Error(`No tests found matching: ${test}`) + + const results = [] + const currentSteps = {} + let currentTestTitle = null + const testFile = testFiles[0] + + const onBefore = (t) => { + const traceDir = getTraceDir(t.title, t.file) + currentTestTitle = t.title + currentSteps[t.title] = [] + results.push({ + test: t.title, + file: t.file, + traceFile: `file://${resolvePath(traceDir, 'trace.md')}`, + status: 'running', + steps: [], + }) + } + + const onAfter = (t) => { + const r = results.find(x => x.test === t.title) + if (r) { + r.status = t.err ? 'failed' : 'completed' + if (t.err) r.error = t.err.message + } + currentTestTitle = null + } + + const onStepAfter = (step) => { + if (!currentTestTitle || !currentSteps[currentTestTitle]) return + currentSteps[currentTestTitle].push({ + step: step.toString(), + status: step.status, + time: step.endTime - step.startTime, + }) + const r = results.find(x => x.test === currentTestTitle) + if (r) r.steps = [...currentSteps[currentTestTitle]] + } + + event.dispatcher.on(event.test.before, onBefore) + event.dispatcher.on(event.test.after, onAfter) + event.dispatcher.on(event.step.after, onStepAfter) + + try { + await Promise.race([ + (async () => { + await codecept.bootstrap() + await codecept.run(testFile) + })(), + new Promise((_, reject) => setTimeout(() => reject(new Error(`Timeout after ${timeout}ms`)), timeout)), + ]) + } catch (error) { + const lastRunning = results.filter(r => r.status === 'running').pop() + if (lastRunning) { + lastRunning.status = 'failed' + lastRunning.error = error.message + } + } finally { + try { event.dispatcher.removeListener(event.test.before, onBefore) } catch {} + try { event.dispatcher.removeListener(event.test.after, onAfter) } catch {} + try { event.dispatcher.removeListener(event.step.after, onStepAfter) } catch {} + } + + return { content: [{ type: 'text', text: JSON.stringify({ results, stepByStep: true }, null, 2) }] } + }) + } + + default: + throw new Error(`Unknown tool: ${name}`) + } + } catch (error) { + return { + content: [{ type: 'text', text: JSON.stringify({ error: error.message, stack: error.stack }, null, 2) }], + isError: true, + } + } +}) + +async function main() { + const transport = new StdioServerTransport() + await server.connect(transport) +} + +main().catch((error) => { + import('fs').then(fs => { + const logFile = path.resolve(process.cwd(), 'mcp-server-error.log') + fs.appendFileSync(logFile, `${new Date().toISOString()} - ${error.stack}\n`) + }) +}) diff --git a/docs/aitrace.md b/docs/aitrace.md new file mode 100644 index 000000000..e96182d53 --- /dev/null +++ b/docs/aitrace.md @@ -0,0 +1,264 @@ +--- +permalink: /aitrace +title: AI Trace Plugin +--- + +# AI Trace Plugin + +AI Trace Plugin generates AI-friendly trace files for debugging with AI agents like Claude Code. + +When a test fails, you need to understand what went wrong: what the page looked like, what elements were present, what errors occurred, and what led to the failure. This plugin automatically captures all that information and organizes it in a format optimized for AI analysis. + +## Quick Start + +Enable the plugin in your `codecept.conf.js`: + +```javascript +export const config = { + tests: './*_test.js', + output: './output', + helpers: { + Playwright: { + url: 'https://example.com', + // Optional: Enable HAR/trace for HTTP capture + recordHar: { + mode: 'minimal', + content: 'embed', + }, + trace: 'on', + keepTraceForPassedTests: true, + }, + }, + plugins: { + aiTrace: { + enabled: true, + }, + }, +} +``` + +Run tests: + +```bash +npx codeceptjs run +``` + +After test execution, trace files are created in `output/trace_*/trace.md`. + +## Artifacts Created + +For each test, a `trace_` directory is created with the following files: + +**trace.md** - AI-friendly markdown file with test execution history + +**0000_step_name_screenshot.png** - Screenshot for each step (file names include step names) + +**0000_step_name_page.html** - Full HTML of the page at each step + +**0000_step_name_aria.txt** - ARIA accessibility snapshot (AI-readable structure without HTML noise) + +**0000_step_name_console.json** - Browser console logs + +When HAR or trace recording is enabled in your helper config, links to those files are also included. + +**Note:** Artifact files are named using step names for easier identification (e.g., `0000_I_see_Product_screenshot.png` instead of just `0000_screenshot.png`). + +## Trace File Format + +The `trace.md` file contains a structured execution log with links to all artifacts: + +```markdown +file: /path/to/test.js +name: My test scenario +time: 3.45s +--- + +I am on page "https://example.com" + > navigated to https://example.com/ + > [HTML](./0000_I_am_on_page_https_example.com_page.html) + > [ARIA Snapshot](./0000_I_am_on_page_https_example.com_aria.txt) + > [Screenshot](./0000_I_am_on_page_https_example.com_screenshot.png) + > [Browser Logs](./0000_I_am_on_page_https_example.com_console.json) (7 entries) + > HTTP: see [HAR file](../har/...) for network requests + +I see "Welcome" + > navigated to https://example.com/ + > [HTML](./0001_I_see_Welcome_page.html) + > [ARIA Snapshot](./0001_I_see_Welcome_aria.txt) + > [Screenshot](./0001_I_see_Welcome_screenshot.png) + > [Browser Logs](./0001_I_see_Welcome_console.json) (0 entries) +``` + +Files are named with step descriptions for easier navigation and debugging. + +## Configuration + +### Basic Configuration + +```javascript +plugins: { + aiTrace: { + enabled: true, + } +} +``` + +### Advanced Configuration + +```javascript +plugins: { + aiTrace: { + enabled: true, + + // Artifact capture options + captureHTML: true, // Save HTML for each step + captureARIA: true, // Save ARIA snapshots + captureBrowserLogs: true, // Save console logs + captureHTTP: true, // Links to HAR/trace files + captureDebugOutput: true, // CodeceptJS debug messages + + // Screenshot options + fullPageScreenshots: false, // Full page or viewport only + + // Output options + output: './output', // Where to save traces + deleteSuccessful: false, // Delete traces for passed tests + + // Step filtering + ignoreSteps: [ + /^grab/, // Ignore all grab* steps + /^wait/, // Ignore all wait* steps + ], + } +} +``` + +### Configuration Options + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `enabled` | boolean | `false` | Enable/disable the plugin | +| `captureHTML` | boolean | `true` | Capture HTML for each step | +| `captureARIA` | boolean | `true` | Capture ARIA snapshots | +| `captureBrowserLogs` | boolean | `true` | Capture browser console logs | +| `captureHTTP` | boolean | `true` | Capture HTTP requests (requires HAR/trace) | +| `captureDebugOutput` | boolean | `true` | Capture CodeceptJS debug output | +| `fullPageScreenshots` | boolean | `false` | Use full page screenshots | +| `output` | string | `'./output'` | Directory for trace files | +| `deleteSuccessful` | boolean | `false` | Delete traces for passed tests | +| `ignoreSteps` | array | `[]` | Steps to ignore (regex patterns) | + +## Best Practices + +### Optimize for Failing Tests + +Save disk space by only keeping traces for failed tests: + +```javascript +plugins: { + aiTrace: { + enabled: true, + deleteSuccessful: true, // Only keep failing tests + } +} +``` + +### Ignore Noise + +Don't capture logs for `grab` and `wait` steps: + +```javascript +plugins: { + aiTrace: { + enabled: true, + ignoreSteps: [/^grab/, /^wait/], + } +} +``` + +### Selective Artifact Capture + +Capture only what you need to reduce file sizes: + +```javascript +plugins: { + aiTrace: { + enabled: true, + captureHTML: false, // Skip HTML (saves ~500KB per step) + captureARIA: true, // Keep ARIA (only ~16KB) + captureBrowserLogs: false, // Skip console logs + } +} +``` + +### Enable HTTP Capture + +For network debugging, enable HAR/trace in your helper: + +```javascript +helpers: { + Playwright: { + recordHar: { + mode: 'minimal', + content: 'embed', + }, + trace: 'on', + keepTraceForPassedTests: true, + }, + plugins: { + aiTrace: { + enabled: true, + captureHTTP: true, // Links to HAR/trace files + }, + }, +} +``` + +## Using with AI Agents + +The trace format is optimized for AI agents like Claude Code. When debugging a failing test, just point the AI agent to the `trace.md` file - it will read the file and all linked artifacts automatically to analyze the failure. + +## Troubleshooting + +### No trace files created + +**Possible causes:** +1. Plugin not enabled +2. No steps executed +3. Tests skipped + +**Solution:** +```bash +# Check if plugin is enabled +grep -A 5 "aiTrace" codecept.conf.js + +# Run with verbose output +npx codeceptjs run --verbose +``` + +### ARIA snapshots missing + +**Possible cause:** Helper doesn't support `grabAriaSnapshot` + +**Solution:** Use Playwright or update to latest CodeceptJS + +### HAR files missing + +**Possible cause:** HAR/trace not enabled in helper config + +**Solution:** +```javascript +helpers: { + Playwright: { + recordHar: { mode: 'minimal' }, + trace: 'on', + }, +} +``` + +## Related + +- [AI Features](/ai) - AI-powered testing features +- [Plugins](/plugins) - All available plugins +- [Configuration](/configuration) - General configuration +- [Playwright Helper](/playwright) - Playwright-specific configuration diff --git a/docs/mcp.md b/docs/mcp.md new file mode 100644 index 000000000..70e914e3e --- /dev/null +++ b/docs/mcp.md @@ -0,0 +1,545 @@ +# CodeceptJS MCP Server + +Model Context Protocol (MCP) server for CodeceptJS enables AI agents (like Claude) to interact with and control CodeceptJS tests programmatically. + +## Overview + +The MCP server provides AI agents with tools to: +- List all tests in a CodeceptJS project +- List all available CodeceptJS actions (I.* methods) +- Run arbitrary CodeceptJS code with artifacts capture +- Run specific tests with detailed output +- Run tests step by step for detailed analysis +- Start and stop browser sessions +- Capture screenshots, ARIA snapshots, HTML, and console logs + +## Installation + +Install the MCP SDK dependency: + +```bash +npm install @modelcontextprotocol/sdk +``` + +The MCP server binary is available at `bin/mcp-server.js`. + +## Configuration + +Configure the MCP server in your Claude Desktop or MCP-compatible client configuration: + +### Basic Configuration + +```json +{ + "mcpServers": { + "codeceptjs": { + "command": "node", + "args": ["path/to/codeceptjs/bin/mcp-server.js"] + } + } +} +``` + +With basic configuration, the server looks for `codecept.conf.js` in the current working directory. + +### Configuration with Environment Variables + +Use environment variables to specify the CodeceptJS project directory and config file: + +```json +{ + "mcpServers": { + "codeceptjs": { + "command": "node", + "args": ["path/to/codeceptjs/bin/mcp-server.js"], + "env": { + "CODECEPTJS_CONFIG": "/path/to/your/codecept.conf.js", + "CODECEPTJS_PROJECT_DIR": "/path/to/your/project" + } + } + } +} +``` + +**Environment Variables:** + +| Variable | Description | +|----------|-------------| +| `CODECEPTJS_CONFIG` | Absolute path to the CodeceptJS configuration file | +| `CODECEPTJS_PROJECT_DIR` | Absolute path to the project root directory | + +### Example: Full Claude Desktop Configuration + +```json +{ + "mcpServers": { + "codeceptjs-mcp": { + "command": "node", + "args": ["D:/projects/my-project/node_modules/codeceptjs/bin/mcp-server.js"], + "env": { + "CODECEPTJS_CONFIG": "D:/projects/my-project/codecept.conf.js", + "CODECEPTJS_PROJECT_DIR": "D:/projects/my-project" + } + } + } +} +``` + +## Available Tools + +### list_tests + +List all tests in the CodeceptJS project. + +**Parameters:** +- `config` (optional): Path to codecept.conf.js (default: codecept.conf.js) + +**Returns:** +```json +{ + "count": 5, + "tests": [ + { + "file": "/full/path/to/test/file.js", + "relativePath": "tests/example_test.js" + } + ] +} +``` + +**Example:** +```json +{ + "name": "list_tests", + "arguments": { + "config": "/path/to/codecept.conf.js" + } +} +``` + +### list_actions + +List all available CodeceptJS actions (I.* methods) from enabled helpers and support objects. + +**Parameters:** +- `config` (optional): Path to codecept.conf.js + +**Returns:** +```json +{ + "count": 120, + "actions": [ + { + "helper": "Playwright", + "action": "amOnPage", + "signature": "I.amOnPage(url)" + }, + { + "helper": "Playwright", + "action": "click", + "signature": "I.click(locator, context)" + } + ] +} +``` + +### run_code + +Run arbitrary CodeceptJS code. Returns status, ARIA snapshot, URL, console logs, and HTML. + +**Parameters:** +- `code` (required): CodeceptJS code to execute +- `timeout` (optional): Timeout in milliseconds (default: 60000) +- `config` (optional): Path to codecept.conf.js +- `saveArtifacts` (optional): Save artifacts like ARIA, URL, console logs, HTML (default: true) + +**Returns:** +```json +{ + "status": "success", + "output": "Code executed successfully", + "error": null, + "artifacts": { + "aria": "main -> \"Welcome\"...", + "url": "http://localhost:8000/", + "consoleLogs": [], + "html": "..." + } +} +``` + +**Example:** +```json +{ + "name": "run_code", + "arguments": { + "code": "await I.amOnPage('/'); await I.see('Welcome');", + "timeout": 30000, + "saveArtifacts": true + } +} +``` + +### run_test + +Run a specific test by name or file path. Uses subprocess to run tests with isolation. + +**Parameters:** +- `test` (required): Test name or file path +- `timeout` (optional): Timeout in milliseconds (default: 60000) +- `config` (optional): Path to codecept.conf.js + +**Returns:** +```json +{ + "meta": { + "exitCode": 0, + "cli": "/path/to/codecept.js", + "root": "/project/root", + "configPath": "/path/to/codecept.conf.js", + "args": ["run", "--config", "...", "--reporter", "json", "test_file.js"], + "resolvedFile": "/full/path/to/test_file.js" + }, + "reporterJson": { + "stats": { + "tests": 3, + "passes": 2, + "failures": 1 + } + }, + "stderr": "", + "rawStdout": "" +} +``` + +**Features:** +- Automatically resolves test names to file paths +- Supports partial test name matching +- Uses json reporter for structured output +- Executes in subprocess for isolation +- Includes stderr for debugging + +**Example:** +```json +{ + "name": "run_test", + "arguments": { + "test": "basic_navigation_test", + "timeout": 60000 + } +} +``` + +### run_step_by_step + +Run a test step by step with detailed step information including timing and status. Generates AI-friendly trace files. + +**Parameters:** +- `test` (required): Test name or file path +- `timeout` (optional): Timeout in milliseconds (default: 60000) +- `config` (optional): Path to codecept.conf.js + +**Returns:** +```json +{ + "stepByStep": true, + "results": [ + { + "test": "Navigate to homepage", + "file": "/path/to/test.js", + "traceFile": "file:///output/trace_Test_Name_abc123/trace.md", + "status": "completed", + "steps": [ + { + "step": "I.amOnPage(\"/\")", + "status": "passed", + "time": 150 + }, + { + "step": "I.seeInTitle(\"Test App\")", + "status": "passed", + "time": 50 + } + ] + } + ] +} +``` + +**Trace Files:** +- Generated in `{output_dir}/trace_{TestName}_{hash}/` +- Includes screenshots (PNG), page HTML, ARIA snapshots, console logs +- `trace.md` file provides structured summary for AI analysis +- Named with test title and hash for uniqueness + +**Example:** +```json +{ + "name": "run_step_by_step", + "arguments": { + "test": "authentication_test", + "timeout": 90000 + } +} +``` + +### start_browser + +Start the browser session (initializes CodeceptJS container). + +**Parameters:** +- `config` (optional): Path to codecept.conf.js + +**Returns:** +```json +{ + "status": "Browser started successfully" +} +``` + +**Note:** Browser is automatically started on first code execution. This tool is useful for pre-initialization. + +### stop_browser + +Stop the browser session and cleanup resources. + +**Parameters:** +- None + +**Returns:** +```json +{ + "status": "Browser stopped successfully" +} +``` + +**Note:** Useful for releasing resources between long-running sessions. + +## Testing + +### Run MCP Server Tests + +The MCP server includes a comprehensive test suite: + +```bash +node test/mcp/mcp_server_test.js +``` + +Tests cover: +- Tool listing and schema validation +- Test enumeration +- Action listing +- Code execution with artifacts +- Test execution (run_test) +- Step-by-step execution +- Browser lifecycle +- Error handling + +### Run Demo Tests with MCP + +**Important: Start the test web server first!** + +The MCP test scenarios require a web server running on port 8000. Start it in a separate terminal: + +```bash +# Option 1: Using http-server (recommended) +cd test/mcp +npx http-server -p 8000 + +# Option 2: Using Python +cd test/mcp +python -m http.server 8000 + +# Option 3: Using PHP +cd test/mcp +php -S localhost:8000 +``` + +The server will start at http://127.0.0.1:8000 + +**Keep this terminal open** while running tests through MCP/Claude. + +Once the server is running, you can use Claude to run tests: + +``` +"List all tests" +"Run basic navigation test" +"Run form interaction test step by step" +``` + +**Note:** If tests fail with ERR_CONNECTION_REFUSED, make sure the web server is running on port 8000. + +## Trace Files for AI Debugging + +When using `run_step_by_step`, the server generates trace files that provide rich context for AI agents: + +### Trace File Structure + +``` +output/ +└── trace_Test_Name_abc123/ + ├── 0000_screenshot.png # Screenshot after step 0 + ├── 0000_page.html # HTML snapshot after step 0 + ├── 0000_aria.txt # ARIA snapshot after step 0 + ├── 0000_console.json # Console logs after step 0 + ├── 0001_screenshot.png # Screenshot after step 1 + ├── 0001_page.html + ├── 0001_aria.txt + ├── 0001_console.json + └── trace.md # AI-friendly summary +``` + +### Using Trace Files with AI + +The `trace.md` file provides structured information perfect for AI analysis: + +```markdown +# Test: Login functionality + +**Status**: failed +**File**: tests/login_test.js + +## Steps + +1. **I.amOnPage("/login")** - passed (150ms) +2. **I.fillField("#username", "user")** - passed (80ms) +3. **I.fillField("#password", "pass")** - passed (75ms) +4. **I.click("#login")** - passed (100ms) +5. **I.see("Welcome")** - failed (50ms) + +## Error + +Element "Welcome" not found + +## Artifacts + +- Screenshot: 0005_screenshot.png +- HTML: 0005_page.html +- ARIA: 0005_aria.txt +``` + +AI agents can use these artifacts to: +- Visualize what the test saw at each step +- Analyze page structure via ARIA +- Debug issues using HTML snapshots +- Identify errors from console logs + +## Architecture + +### Request Flow + +1. MCP Client sends JSON-RPC request via stdin/stdout +2. Server processes request and calls appropriate tool +3. Tool executes CodeceptJS code or runs tests +4. Results formatted as JSON and returned +5. MCP Client receives response + +### Session Management + +- **Initialization**: CodeceptJS container initialized on first request +- **Browser**: Started once and reused across requests +- **Locking**: `run_test` uses locking to prevent concurrent test runs +- **Cleanup**: `stop_browser` releases all resources + +### Error Handling + +- All errors returned as JSON with error message and stack +- Invalid tools return error response +- Test failures included in results (not thrown) +- Timeout protection on all long-running operations + +## Troubleshooting + +### MCP Server Not Starting + +- Ensure `@modelcontextprotocol/sdk` is installed +- Check Node.js version (requires Node.js 16+) +- Verify the path to mcp-server.js in your MCP client config +- Check file permissions + +### Configuration Not Found + +- Set `CODECEPTJS_CONFIG` environment variable to absolute path of your config file +- Set `CODECEPTJS_PROJECT_DIR` environment variable to your project directory +- Use absolute paths in environment variables (e.g., `D:/projects/my-project/codecept.conf.js`) +- Verify config file exists and is valid JavaScript + +### Tests Not Found + +- Verify you're in the correct working directory +- Check that `codecept.conf.js` exists +- Use absolute paths for tests if relative paths don't work +- Check test patterns in config file match your test files + +### Browser Launch Issues + +- Ensure browser dependencies are installed (Chromium for Playwright) +- Check if browser is already running +- Verify `show: false` in config (headless mode recommended) +- Check firewall/proxy settings + +### Tests Stuck or Timing Out + +- Increase timeout parameter (default 60s) +- Check if web server is running (for tests that need it) +- Disable video recording and other heavy features +- Use `run_test` instead of `run_step_by_step` for faster execution + +## Advanced Usage + +### Custom Test Patterns + +```javascript +// In codecept.conf.js +export const config = { + tests: './tests/**/*_test.js', + // ... rest of config +} +``` + +### AI-Friendly Trace Integration + +For best results with AI agents: + +1. **Enable aiTrace plugin** in config (automatically enabled for `run_step_by_step`) +2. **Use descriptive test names** for better trace file organization +3. **Keep tests focused** - one scenario per test for clearer traces +4. **Add assertions with clear messages** - better error reporting + +### Running Tests from Different Directories + +```json +{ + "mcpServers": { + "codeceptjs": { + "command": "node", + "args": ["node_modules/codeceptjs/bin/mcp-server.js"], + "env": { + "CODECEPTJS_CONFIG": "/absolute/path/to/codecept.conf.js", + "CODECEPTJS_PROJECT_DIR": "/absolute/path/to/project" + } + } + } +} +``` + +## Security Considerations + +- MCP server runs with same permissions as calling process +- `run_code` allows arbitrary CodeceptJS execution - use in trusted environments only +- Test files should validate input if exposed to external systems +- Environment variables may contain sensitive paths - secure accordingly + +## Contributing + +When contributing to MCP server: + +1. Add tests for new tools in `test/mcp/mcp_server_test.js` +2. Update this documentation with new tools/parameters +3. Ensure error handling is consistent +4. Test with both Playwright and Puppeteer helpers +5. Verify trace files are generated correctly for `run_step_by_step` + +## License + +MIT diff --git a/docs/plugins.md b/docs/plugins.md index 1a279cd37..408db3c16 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -63,6 +63,118 @@ exports.config = { Returns **void** +## aiTrace + +Generates AI-friendly trace files for debugging with AI agents like Claude Code. + +When a test fails, you need to understand what went wrong. This plugin automatically captures comprehensive information about test execution - screenshots, HTML, ARIA snapshots, console logs, and HTTP requests - and organizes it in a format optimized for AI analysis. + +The generated trace files are structured markdown documents that AI agents can easily parse to understand test context and provide debugging insights. + +#### Usage + +Enable this plugin in your config: + +```js +// in codecept.conf.js +exports.config = { + plugins: { + aiTrace: { + enabled: true + } + } +} +``` + +#### Configuration + +* `deleteSuccessful` (boolean) - delete traces for successfully executed tests. Default: false. +* `fullPageScreenshots` (boolean) - should full page screenshots be used. Default: false. +* `output` (string) - a directory where traces should be stored. Default: `output`. +* `captureHTML` (boolean) - capture HTML for each step. Default: true. +* `captureARIA` (boolean) - capture ARIA snapshot for each step. Default: true. +* `captureBrowserLogs` (boolean) - capture browser console logs. Default: true. +* `captureHTTP` (boolean) - capture HTTP requests (requires `trace` or `recordHar` enabled in helper config). Default: true. +* `captureDebugOutput` (boolean) - capture CodeceptJS debug output. Default: true. +* `ignoreSteps` (array) - steps to ignore in trace. Array of RegExps is expected. Default: []. + +#### Artifacts Created + +For each test, a `trace_` directory is created with: + +* **trace.md** - AI-friendly markdown file with test execution history +* **0000_step_name_screenshot.png** - screenshot for each step (named with step description) +* **0000_step_name_page.html** - full HTML of the page at each step +* **0000_step_name_aria.txt** - ARIA accessibility snapshot (AI-readable structure) +* **0000_step_name_console.json** - browser console logs + +Artifact files include step names for easier identification (e.g., `0000_I_see_Product_screenshot.png`). + +When HAR or trace recording is enabled in your helper config, links to those files are also included. + +#### Example Output + +```markdown +file: /path/to/test.js +name: My test scenario +time: 3.45s +--- + +I am on page "https://example.com" + > navigated to https://example.com/ + > [HTML](./0000_I_am_on_page_https_example.com_page.html) + > [ARIA Snapshot](./0000_I_am_on_page_https_example.com_aria.txt) + > [Screenshot](./0000_I_am_on_page_https_example.com_screenshot.png) + > [Browser Logs](./0000_I_am_on_page_https_example.com_console.json) (7 entries) + > HTTP: see [HAR file](../har/...) for network requests + +I see "Welcome" + > navigated to https://example.com/ + > [HTML](./0001_I_see_Welcome_page.html) + > [ARIA Snapshot](./0001_I_see_Welcome_aria.txt) + > [Screenshot](./0001_I_see_Welcome_screenshot.png) + > [Browser Logs](./0001_I_see_Welcome_console.json) (0 entries) +``` + +Files are named with step descriptions for easier identification. + +#### Best Practices + +**Save disk space** - Only keep traces for failed tests: + +```js +aiTrace: { + enabled: true, + deleteSuccessful: true +} +``` + +**Ignore noise** - Don't capture logs for `grab` and `wait` steps: + +```js +aiTrace: { + enabled: true, + ignoreSteps: [/^grab/, /^wait/] +} +``` + +**Reduce file sizes** - Capture only what you need: + +```js +aiTrace: { + enabled: true, + captureHTML: false, // Skip HTML (saves ~500KB per step) + captureARIA: true, // Keep ARIA (only ~16KB) + captureBrowserLogs: false // Skip console logs +} +``` + +### Parameters + +* `config` **[Object][1]** Plugin configuration (optional, default `{}`) + +Returns **void** + ## auth Logs user in for the first test and reuses session for next tests. diff --git a/examples/codecept.config.js b/examples/codecept.config.js index 399e6a248..c59e980be 100644 --- a/examples/codecept.config.js +++ b/examples/codecept.config.js @@ -60,6 +60,14 @@ export const config = { subtitles: { enabled: true, }, + aiTrace: { + enabled: true, + captureHTML: true, + captureARIA: true, + captureBrowserLogs: true, + captureHTTP: true, + ignoreSteps: [/^grab/, /^wait/], + }, }, tests: './*_test.js', diff --git a/lib/plugin/aiTrace.js b/lib/plugin/aiTrace.js new file mode 100644 index 000000000..e0b114a34 --- /dev/null +++ b/lib/plugin/aiTrace.js @@ -0,0 +1,452 @@ +import crypto from 'crypto' +import fs from 'fs' +import { mkdirp } from 'mkdirp' +import path from 'path' +import { fileURLToPath } from 'url' + +import Container from '../container.js' +import recorder from '../recorder.js' +import event from '../event.js' +import output from '../output.js' +import { deleteDir, clearString } from '../utils.js' +import colors from 'chalk' + +const supportedHelpers = Container.STANDARD_ACTING_HELPERS + +const defaultConfig = { + deleteSuccessful: false, + fullPageScreenshots: false, + output: global.output_dir, + captureHTML: true, + captureARIA: true, + captureBrowserLogs: true, + captureHTTP: true, + captureDebugOutput: true, + ignoreSteps: [], +} + +/** + * + * Generates AI-friendly trace files for debugging with AI agents. + * This plugin creates a markdown file with test execution logs and links to all artifacts + * (screenshots, HTML, ARIA snapshots, browser logs, HTTP requests) for each step. + * + * #### Configuration + * + * ```js + * "plugins": { + * "aiTrace": { + * "enabled": true + * } + * } + * ``` + * + * Possible config options: + * + * * `deleteSuccessful`: delete traces for successfully executed tests. Default: false. + * * `fullPageScreenshots`: should full page screenshots be used. Default: false. + * * `output`: a directory where traces should be stored. Default: `output`. + * * `captureHTML`: capture HTML for each step. Default: true. + * * `captureARIA`: capture ARIA snapshot for each step. Default: true. + * * `captureBrowserLogs`: capture browser console logs. Default: true. + * * `captureHTTP`: capture HTTP requests (requires `trace` or `recordHar` enabled in helper config). Default: true. + * * `captureDebugOutput`: capture CodeceptJS debug output. Default: true. + * * `ignoreSteps`: steps to ignore in trace. Array of RegExps is expected. + * + * @param {*} config + */ +export default function (config) { + const helpers = Container.helpers() + let helper + + config = Object.assign(defaultConfig, config) + + for (const helperName of supportedHelpers) { + if (Object.keys(helpers).indexOf(helperName) > -1) { + helper = helpers[helperName] + } + } + + if (!helper) { + output.warn('aiTrace plugin: No supported helper found (Playwright, Puppeteer, WebDriver). Plugin disabled.') + return + } + + let dir + let stepNum + let steps = [] + let debugOutput = [] + let error + let savedSteps = new Set() + let currentTest = null + let testStartTime + let currentUrl = null + let testFailed = false + let firstFailedStepSaved = false + + const reportDir = config.output ? path.resolve(global.codecept_dir, config.output) : defaultConfig.output + + if (config.captureDebugOutput) { + const originalDebug = output.debug + output.debug = function (...args) { + debugOutput.push(args.join(' ')) + originalDebug.apply(output, args) + } + } + + event.dispatcher.on(event.suite.before, suite => { + stepNum = -1 + }) + + event.dispatcher.on(event.test.before, test => { + let title + try { + title = test.fullTitle ? test.fullTitle() : test.title + } catch (err) { + title = test.title + } + const testTitle = clearString(title).slice(0, 200) + const uniqueHash = crypto + .createHash('sha256') + .update(test.file + test.title) + .digest('hex') + .slice(0, 8) + dir = path.join(reportDir, `trace_${testTitle}_${uniqueHash}`) + mkdirp.sync(dir) + deleteDir(dir) + mkdirp.sync(dir) + stepNum = 0 + error = null + steps = [] + debugOutput = [] + savedSteps.clear() + currentTest = test + testStartTime = Date.now() + currentUrl = null + testFailed = false + firstFailedStepSaved = false + }) + + event.dispatcher.on(event.step.after, async step => { + if (!currentTest) return + if (step.status === 'failed') { + testFailed = true + } + if (step.status === 'queued' && testFailed) { + output.debug(`aiTrace: Skipping queued step "${step.toString()}" - testFailed: ${testFailed}`) + return + } + if (step.status === 'failed' && firstFailedStepSaved) { + output.debug(`aiTrace: Skipping failed step "${step.toString()}" - already handled by step.failed event`) + return + } + recorder.add(`save artifacts for step ${step.toString()}`, async () => { + try { + await persistStep(step) + } catch (err) { + output.debug(`aiTrace: Error saving step: ${err.message}`) + } + }, true) + }) + + event.dispatcher.on(event.step.failed, async step => { + if (!currentTest) return + if (step.status === 'queued' && testFailed) { + output.debug(`aiTrace: Skipping queued failed step "${step.toString()}" - testFailed: ${testFailed}`) + return + } + if (firstFailedStepSaved) { + output.debug(`aiTrace: Skipping subsequent failed step "${step.toString()}" - already saved first failed step`) + return + } + + const stepKey = step.toString() + if (savedSteps.has(stepKey)) { + const existingStep = steps.find(s => s.step === stepKey) + if (!existingStep) { + output.debug(`aiTrace: Step "${stepKey}" marked as saved but not found in steps array`) + return + } + existingStep.status = 'failed' + + try { + await captureArtifactsForStep(step, existingStep, existingStep.prefix) + } catch (err) { + output.debug(`aiTrace: Error updating failed step: ${err.message}`) + } + } else { + if (stepNum === -1) return + if (isStepIgnored(step)) return + if (step.metaStep && step.metaStep.name === 'BeforeSuite') return + + const stepPrefix = generateStepPrefix(step, stepNum) + stepNum++ + + const stepData = { + step: stepKey, + status: 'failed', + prefix: stepPrefix, + artifacts: {}, + meta: {}, + debugOutput: [], + } + + if (step.startTime && step.endTime) { + stepData.meta.duration = ((step.endTime - step.startTime) / 1000).toFixed(2) + 's' + } + + savedSteps.add(stepKey) + steps.push(stepData) + firstFailedStepSaved = true + + try { + await captureArtifactsForStep(step, stepData, stepPrefix) + } catch (err) { + output.debug(`aiTrace: Error capturing failed step artifacts: ${err.message}`) + } + } + }) + + event.dispatcher.on(event.test.passed, test => { + if (config.deleteSuccessful) { + deleteDir(dir) + return + } + persist(test, 'passed') + }) + + event.dispatcher.on(event.test.failed, (test, _err, hookName) => { + if (hookName === 'BeforeSuite' || hookName === 'AfterSuite') { + return + } + persist(test, 'failed') + }) + + async function persistStep(step) { + if (stepNum === -1) return + if (isStepIgnored(step)) return + if (step.metaStep && step.metaStep.name === 'BeforeSuite') return + + const stepKey = step.toString() + + if (savedSteps.has(stepKey)) { + const existingStep = steps.find(s => s.step === stepKey) + if (existingStep && step.status === 'failed') { + existingStep.status = 'failed' + step.artifacts = {} + await captureArtifactsForStep(step, existingStep, existingStep.prefix) + } + return + } + savedSteps.add(stepKey) + + const stepPrefix = generateStepPrefix(step, stepNum) + stepNum++ + + const stepData = { + step: step.toString(), + status: step.status, + prefix: stepPrefix, + artifacts: {}, + meta: {}, + debugOutput: [], + } + + if (step.startTime && step.endTime) { + stepData.meta.duration = ((step.endTime - step.startTime) / 1000).toFixed(2) + 's' + } + + if (config.captureDebugOutput && debugOutput.length > 0) { + stepData.debugOutput = [...debugOutput] + debugOutput = [] + } + + await captureArtifactsForStep(step, stepData, stepPrefix) + steps.push(stepData) + } + + async function captureArtifactsForStep(step, stepData, stepPrefix) { + if (!step.artifacts) { + step.artifacts = {} + } + + let browserAvailable = true + + try { + try { + if (helper.grabCurrentUrl) { + const url = await helper.grabCurrentUrl() + stepData.meta.url = url + currentUrl = url + } + } catch (err) { + browserAvailable = false + output.debug(`aiTrace: Browser unavailable, partial artifact capture: ${err.message}`) + } + + if (!step.artifacts?.screenshot) { + try { + const screenshotFile = `${stepPrefix}_screenshot.png` + const screenshotPath = path.join(dir, screenshotFile) + await helper.saveScreenshot(screenshotPath, config.fullPageScreenshots) + + stepData.artifacts.screenshot = screenshotFile + step.artifacts.screenshot = screenshotPath + } catch (err) { + output.debug(`aiTrace: Could not save screenshot: ${err.message}`) + } + } + + // Save HTML + if (config.captureHTML && helper.grabSource && browserAvailable) { + if (!step.artifacts?.html) { + try { + const html = await helper.grabSource() + const htmlFile = `${stepPrefix}_page.html` + fs.writeFileSync(path.join(dir, htmlFile), html) + stepData.artifacts.html = htmlFile + } catch (err) { + output.debug(`aiTrace: Could not capture HTML: ${err.message}`) + } + } else { + stepData.artifacts.html = step.artifacts.html + } + } + + // Save ARIA snapshot + if (config.captureARIA && helper.grabAriaSnapshot && browserAvailable) { + try { + const aria = await helper.grabAriaSnapshot() + const ariaFile = `${stepPrefix}_aria.txt` + fs.writeFileSync(path.join(dir, ariaFile), aria) + stepData.artifacts.aria = ariaFile + } catch (err) { + output.debug(`aiTrace: Could not capture ARIA snapshot: ${err.message}`) + } + } + + // Save browser logs + if (config.captureBrowserLogs && helper.grabBrowserLogs && browserAvailable) { + try { + const logs = await helper.grabBrowserLogs() + const logsFile = `${stepPrefix}_console.json` + fs.writeFileSync(path.join(dir, logsFile), JSON.stringify(logs || [], null, 2)) + stepData.artifacts.console = logsFile + stepData.meta.consoleCount = logs ? logs.length : 0 + } catch (err) { + output.debug(`aiTrace: Could not capture browser logs: ${err.message}`) + } + } + } catch (err) { + output.plugin(`aiTrace: Can't save step artifacts: ${err}`) + } + } + + function persist(test, status) { + if (!steps.length) { + output.debug('aiTrace: No steps to save in trace') + return + } + + const testDuration = ((Date.now() - testStartTime) / 1000).toFixed(2) + + let markdown = `file: ${test.file || 'unknown'}\n` + markdown += `name: ${test.title}\n` + markdown += `time: ${testDuration}s\n` + markdown += `---\n\n` + + if (status === 'failed') { + if (test.art && test.art.message) { + markdown += `Error: ${test.art.message}\n\n` + } + if (test.art && test.art.stack) { + markdown += `${test.art.stack}\n\n` + } + markdown += `---\n\n` + } + + if (config.captureDebugOutput && debugOutput.length > 0) { + markdown += `CodeceptJS Debug Output:\n\n` + debugOutput.forEach(line => { + markdown += `> ${line}\n` + }) + markdown += `\n---\n\n` + } + + steps.forEach((stepData, index) => { + const stepAnchor = clearString(stepData.step).replace(/[^a-zA-Z0-9_-]/g, '_').slice(0, 50) + markdown += `### Step ${index + 1}: ${stepData.step}\n` + markdown += `\n` + + if (stepData.meta.duration) { + markdown += ` > duration: ${stepData.meta.duration}\n` + } + + if (stepData.meta.url) { + markdown += ` > navigated to ${stepData.meta.url}\n` + } + + if (config.captureDebugOutput && stepData.debugOutput && stepData.debugOutput.length > 0) { + stepData.debugOutput.forEach(line => { + markdown += ` > ${line}\n` + }) + } + + if (stepData.artifacts.html) { + markdown += ` > [HTML](./${stepData.artifacts.html})\n` + } + + if (stepData.artifacts.aria) { + markdown += ` > [ARIA Snapshot](./${stepData.artifacts.aria})\n` + } + + if (stepData.artifacts.screenshot) { + markdown += ` > [Screenshot](./${stepData.artifacts.screenshot})\n` + } + + if (stepData.artifacts.console) { + const count = stepData.meta.consoleCount || 0 + markdown += ` > [Browser Logs](./${stepData.artifacts.console}) (${count} entries)\n` + } + + if (config.captureHTTP) { + if (test.artifacts && test.artifacts.har) { + const harPath = path.relative(reportDir, test.artifacts.har) + markdown += ` > HTTP: see [HAR file](../${harPath}) for network requests\n` + } else if (test.artifacts && test.artifacts.trace) { + const tracePath = path.relative(reportDir, test.artifacts.trace) + markdown += ` > HTTP: see [Playwright trace](../${tracePath}) for network requests\n` + } + } + + markdown += `\n` + }) + + const traceFile = path.join(dir, 'trace.md') + fs.writeFileSync(traceFile, markdown) + + output.print(`🤖 AI Trace: ${colors.white.bold(`file://${traceFile}`)}`) + + if (!test.artifacts) test.artifacts = {} + test.artifacts.aiTrace = traceFile + } + + function isStepIgnored(step) { + if (!config.ignoreSteps) return false + for (const pattern of config.ignoreSteps || []) { + if (step.name.match(pattern)) return true + } + return false + } + + function generateStepPrefix(step, index) { + const stepName = step.toString() + const cleanedName = clearString(stepName) + .replace(/[^a-zA-Z0-9_-]/g, '_') + .replace(/_{2,}/g, '_') + .slice(0, 80) + .trim() + + return `${String(index).padStart(4, '0')}_${cleanedName}` + } +} diff --git a/lib/plugin/stepByStepReport.js b/lib/plugin/stepByStepReport.js index ce53125af..ee1cb52ec 100644 --- a/lib/plugin/stepByStepReport.js +++ b/lib/plugin/stepByStepReport.js @@ -207,7 +207,11 @@ export default function (config) { stepNum++ slides[fileName] = step try { - await helper.saveScreenshot(path.join(dir, fileName), config.fullPageScreenshots) + const screenshotPath = path.join(dir, fileName) + await helper.saveScreenshot(screenshotPath, config.fullPageScreenshots) + + step.artifacts = step.artifacts || {} + step.artifacts.screenshot = screenshotPath } catch (err) { output.plugin(`Can't save step screenshot: ${err}`) error = err diff --git a/package.json b/package.json index 067cb5600..649333186 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,8 @@ "./store": "./lib/store.js" }, "bin": { - "codeceptjs": "./bin/codecept.js" + "codeceptjs": "./bin/codecept.js", + "codeceptjs-mcp": "./bin/mcp-server.js" }, "repository": "Codeception/codeceptjs", "scripts": { @@ -90,6 +91,7 @@ "@cucumber/cucumber-expressions": "18", "@cucumber/gherkin": "38.0.0", "@cucumber/messages": "32.0.1", + "@modelcontextprotocol/sdk": "^1.26.0", "@xmldom/xmldom": "0.9.8", "acorn": "8.15.0", "ai": "^6.0.43", diff --git a/test/unit/mcpServer_test.js b/test/unit/mcpServer_test.js new file mode 100644 index 000000000..3dba334a2 --- /dev/null +++ b/test/unit/mcpServer_test.js @@ -0,0 +1,405 @@ +import { expect } from 'chai' +import sinon from 'sinon' +import { fileURLToPath } from 'url' +import { dirname, resolve, join } from 'path' +import { existsSync, mkdirSync, rmSync } from 'fs' +import { createHash } from 'crypto' + +const __filename = fileURLToPath(import.meta.url) +const __dirname = dirname(__filename) + +function extractFilesFromListJson(json) { + if (!json) return [] + if (Array.isArray(json)) return json.map(String) + if (Array.isArray(json.tests)) return json.tests.map(String) + if (Array.isArray(json.files)) return json.files.map(String) + if (Array.isArray(json.testFiles)) return json.testFiles.map(String) + return [] +} + +function looksLikePath(v) { + return typeof v === 'string' && ( + v.includes('/') || v.includes('\\') || + v.endsWith('.js') || v.endsWith('.ts') + ) +} + +function normalizePath(p) { + return String(p).replace(/\\/g, '/') +} + +function clearString(str) { + return str.replace(/[^a-zA-Z0-9]/g, '_') +} + +function getTraceDir(testTitle, testFile) { + const hash = createHash('sha256').update(testFile + testTitle).digest('hex').slice(0, 8) + const cleanTitle = clearString(testTitle).slice(0, 200) + const outputDir = global.output_dir || resolve(process.cwd(), 'output') + return resolve(outputDir, `trace_${cleanTitle}_${hash}`) +} + +describe('MCP Server Utilities', () => { + describe('extractFilesFromListJson', () => { + it('should extract files from array', () => { + const json = ['/path/to/test1.js', '/path/to/test2.js'] + const result = extractFilesFromListJson(json) + expect(result).to.deep.equal(['/path/to/test1.js', '/path/to/test2.js']) + }) + + it('should extract files from object with tests property', () => { + const json = { tests: ['/path/to/test1.js', '/path/to/test2.js'] } + const result = extractFilesFromListJson(json) + expect(result).to.deep.equal(['/path/to/test1.js', '/path/to/test2.js']) + }) + + it('should extract files from object with files property', () => { + const json = { files: ['/path/to/test1.js'] } + const result = extractFilesFromListJson(json) + expect(result).to.deep.equal(['/path/to/test1.js']) + }) + + it('should extract files from object with testFiles property', () => { + const json = { testFiles: ['/path/to/test1.js'] } + const result = extractFilesFromListJson(json) + expect(result).to.deep.equal(['/path/to/test1.js']) + }) + + it('should return empty array for null input', () => { + const result = extractFilesFromListJson(null) + expect(result).to.deep.equal([]) + }) + + it('should return empty array for invalid input', () => { + const result = extractFilesFromListJson({ invalid: true }) + expect(result).to.deep.equal([]) + }) + }) + + describe('looksLikePath', () => { + it('should return true for path with forward slashes', () => { + expect(looksLikePath('tests/example_test.js')).to.be.true + }) + + it('should return true for path with backslashes', () => { + expect(looksLikePath('tests\\example_test.js')).to.be.true + }) + + it('should return true for .js file', () => { + expect(looksLikePath('example_test.js')).to.be.true + }) + + it('should return true for .ts file', () => { + expect(looksLikePath('example_test.ts')).to.be.true + }) + + it('should return false for test name', () => { + expect(looksLikePath('my test')).to.be.false + }) + + it('should return false for simple string', () => { + expect(looksLikePath('login')).to.be.false + }) + }) + + describe('normalizePath', () => { + it('should convert backslashes to forward slashes', () => { + const result = normalizePath('path\\to\\file.js') + expect(result).to.equal('path/to/file.js') + }) + + it('should handle mixed slashes', () => { + const result = normalizePath('path/to\\file.js') + expect(result).to.equal('path/to/file.js') + }) + + it('should handle paths with no slashes', () => { + const result = normalizePath('file.js') + expect(result).to.equal('file.js') + }) + + it('should convert string to string', () => { + const result = normalizePath(123) + expect(result).to.equal('123') + }) + }) + + describe('clearString', () => { + it('should replace special characters with underscore', () => { + expect(clearString('test-name')).to.equal('test_name') + }) + + it('should handle multiple special characters', () => { + expect(clearString('test@#$%name')).to.equal('test____name') + }) + + it('should preserve alphanumeric characters', () => { + expect(clearString('test123Name')).to.equal('test123Name') + }) + + it('should replace spaces with underscores', () => { + expect(clearString('test name')).to.equal('test_name') + }) + + it('should handle empty string', () => { + expect(clearString('')).to.equal('') + }) + }) + + describe('getTraceDir', () => { + it('should generate unique trace directory name', () => { + const testFile = '/path/to/test.js' + const testTitle = 'My Test' + + const result = getTraceDir(testTitle, testFile) + + expect(result).to.be.a('string') + expect(result).to.include('trace_') + }) + + it('should use hash for uniqueness', () => { + const testFile = '/path/to/test.js' + const testTitle = 'My Test' + + const result1 = getTraceDir(testTitle, testFile) + const result2 = getTraceDir(testTitle, testFile) + + expect(result1).to.equal(result2) + }) + + it('should sanitize test title in directory name', () => { + const testFile = '/path/to/test.js' + const testTitle = 'Test: Special@#$ Characters' + + const result = getTraceDir(testTitle, testFile) + + expect(result).to.not.include('@') + expect(result).to.not.include('#') + expect(result).to.not.include('$') + expect(result).to.not.include(':') + }) + }) +}) + +describe('MCP Server Integration', () => { + let testOutputDir + + beforeEach(() => { + testOutputDir = resolve(__dirname, '../output/mcp-test') + if (existsSync(testOutputDir)) { + rmSync(testOutputDir, { recursive: true, force: true }) + } + mkdirSync(testOutputDir, { recursive: true }) + + sinon.stub(process, 'stdout').value({ + write: sinon.stub().returns(true), + }) + + sinon.stub(process, 'stderr').value({ + write: sinon.stub().returns(true), + }) + + sinon.stub(process, 'stdin').value({ + write: sinon.stub(), + }) + }) + + afterEach(() => { + sinon.restore() + + if (existsSync(testOutputDir)) { + try { + rmSync(testOutputDir, { recursive: true, force: true }) + } catch (e) { + } + } + }) + + describe('Tool Response Format', () => { + it('should return properly formatted tool response', () => { + const response = { + content: [ + { + type: 'text', + text: JSON.stringify({ status: 'success', output: 'test' }, null, 2), + }, + ], + } + + expect(response).to.have.property('content') + expect(response.content).to.be.an('array') + expect(response.content[0]).to.have.property('type', 'text') + expect(response.content[0]).to.have.property('text') + + const parsed = JSON.parse(response.content[0].text) + expect(parsed).to.have.property('status', 'success') + }) + + it('should return error response properly formatted', () => { + const errorResponse = { + content: [ + { + type: 'text', + text: JSON.stringify({ error: 'Test error', stack: 'error stack' }, null, 2), + }, + ], + isError: true, + } + + expect(errorResponse).to.have.property('isError', true) + expect(errorResponse.content[0].type).to.equal('text') + + const parsed = JSON.parse(errorResponse.content[0].text) + expect(parsed).to.have.property('error') + expect(parsed).to.have.property('stack') + }) + }) + + describe('Concurrency Control', () => { + it('should serialize execution using lock', async function () { + this.timeout(10000) + + let executionOrder = [] + let lock = Promise.resolve() + + async function withLock(fn) { + const prev = lock + let release + lock = new Promise(r => (release = r)) + + await prev + try { + return await fn() + } finally { + release() + } + } + + const task1 = withLock(async () => { + executionOrder.push('start1') + await new Promise(resolve => setTimeout(resolve, 100)) + executionOrder.push('end1') + }) + + const task2 = withLock(async () => { + executionOrder.push('start2') + await new Promise(resolve => setTimeout(resolve, 50)) + executionOrder.push('end2') + }) + + const task3 = withLock(async () => { + executionOrder.push('start3') + await new Promise(resolve => setTimeout(resolve, 50)) + executionOrder.push('end3') + }) + + await Promise.all([task1, task2, task3]) + + expect(executionOrder).to.deep.equal([ + 'start1', + 'end1', + 'start2', + 'end2', + 'start3', + 'end3', + ]) + }) + }) + + describe('Error Handling', () => { + it('should format error messages correctly', () => { + const error = new Error('Test error message') + const errorResponse = { + content: [ + { + type: 'text', + text: JSON.stringify({ error: error.message, stack: error.stack }, null, 2), + }, + ], + isError: true, + } + + expect(errorResponse.isError).to.be.true + const parsed = JSON.parse(errorResponse.content[0].text) + expect(parsed.error).to.equal('Test error message') + expect(parsed.stack).to.be.a('string') + }) + }) + + describe('Artifact Capture', () => { + it('should include all artifact types when saveArtifacts is true', () => { + const artifacts = { + aria: 'aria snapshot content', + url: 'http://localhost:8000/page', + consoleLogs: [{ level: 'info', text: 'log message' }], + html: 'Page content', + } + + expect(artifacts).to.have.property('aria') + expect(artifacts).to.have.property('url') + expect(artifacts).to.have.property('consoleLogs') + expect(artifacts).to.have.property('html') + }) + + it('should handle missing artifact methods gracefully', () => { + const partialArtifacts = { + url: 'http://localhost:8000/', + } + + expect(partialArtifacts).to.have.property('url') + expect(partialArtifacts).to.not.have.property('aria') + }) + }) + + describe('Test Result Formats', () => { + it('should format step-by-step results correctly', () => { + const results = [ + { + test: 'Login functionality', + file: '/path/to/login_test.js', + traceFile: 'file:///output/trace_Login_abc123/trace.md', + status: 'completed', + steps: [ + { step: 'I.amOnPage("/login")', status: 'passed', time: 150 }, + { step: 'I.fillField("#username", "user")', status: 'passed', time: 80 }, + ], + }, + ] + + expect(results).to.be.an('array') + expect(results[0]).to.have.property('test') + expect(results[0]).to.have.property('file') + expect(results[0]).to.have.property('traceFile') + expect(results[0]).to.have.property('status') + expect(results[0]).to.have.property('steps') + expect(results[0].steps).to.be.an('array') + expect(results[0].steps[0]).to.have.property('step') + expect(results[0].steps[0]).to.have.property('status') + expect(results[0].steps[0]).to.have.property('time') + }) + + it('should format run_test results correctly', () => { + const result = { + meta: { + exitCode: 0, + cli: '/path/to/codecept.js', + root: '/project/root', + configPath: '/path/to/codecept.conf.js', + args: ['run', '--config', '/path/to/codecept.conf.js', '--reporter', 'json', 'test.js'], + resolvedFile: '/full/path/to/test.js', + }, + reporterJson: { + stats: { tests: 3, passes: 2, failures: 1 }, + }, + stderr: '', + rawStdout: '', + } + + expect(result).to.have.property('meta') + expect(result.meta).to.have.property('exitCode', 0) + expect(result.meta).to.have.property('resolvedFile') + expect(result).to.have.property('reporterJson') + expect(result.reporterJson).to.have.property('stats') + }) + }) +}) diff --git a/test/unit/plugin/aiTrace_test.js b/test/unit/plugin/aiTrace_test.js new file mode 100644 index 000000000..e71e97b49 --- /dev/null +++ b/test/unit/plugin/aiTrace_test.js @@ -0,0 +1,417 @@ +import { expect } from 'chai' +import sinon from 'sinon' +import aiTrace from '../../../lib/plugin/aiTrace.js' +import container from '../../../lib/container.js' +import event from '../../../lib/event.js' +import recorder from '../../../lib/recorder.js' +import output from '../../../lib/output.js' +import { createTest } from '../../../lib/mocha/test.js' +import path from 'path' +import fs from 'fs' + +const testsDir = path.join(process.cwd(), 'test/output') + +describe('aiTrace plugin', () => { + let helperStub + + beforeEach(() => { + recorder.reset() + + helperStub = { + saveScreenshot: sinon.stub().resolves(), + grabSource: sinon.stub().resolves(''), + grabAriaSnapshot: sinon.stub().resolves('- region\n- text: Test'), + grabBrowserLogs: sinon.stub().resolves([]), + } + + container.clear({ + Playwright: helperStub, + }) + + sinon.stub(output, 'print') + }) + + afterEach(() => { + sinon.restore() + event.dispatcher.removeAllListeners(event.test.before) + event.dispatcher.removeAllListeners(event.test.after) + event.dispatcher.removeAllListeners(event.test.passed) + event.dispatcher.removeAllListeners(event.test.failed) + event.dispatcher.removeAllListeners(event.step.after) + }) + + it('should save artifacts for each step', async () => { + aiTrace({ + enabled: true, + output: testsDir, + }) + + const test = createTest('test one') + event.dispatcher.emit(event.test.before, test) + await recorder.promise() + + const step = { + name: 'amOnPage', + toString: () => 'I am on page', + meta: { url: 'https://example.com' }, + status: 'success', + startTime: Date.now(), + endTime: Date.now() + 100, + } + + event.dispatcher.emit(event.step.after, step) + await recorder.promise() + + expect(helperStub.saveScreenshot.calledOnce).to.be.true + expect(helperStub.grabSource.calledOnce).to.be.true + expect(helperStub.grabAriaSnapshot.calledOnce).to.be.true + expect(helperStub.grabBrowserLogs.calledOnce).to.be.true + }) + + it('should generate trace on test passed', async () => { + aiTrace({ + enabled: true, + output: testsDir, + }) + + const test = createTest('test one') + test.art = {} + + event.dispatcher.emit(event.test.before, test) + await recorder.promise() + + const step = { + name: 'see', + toString: () => 'I see test', + status: 'success', + startTime: Date.now(), + endTime: Date.now() + 100, + } + + event.dispatcher.emit(event.step.after, step) + await recorder.promise() + + event.dispatcher.emit(event.test.passed, test) + await recorder.promise() + + expect(test.artifacts.aiTrace).to.be.ok + expect(test.artifacts.aiTrace).to.include('trace.md') + }) + + it('should generate trace on test failed', async () => { + aiTrace({ + enabled: true, + output: testsDir, + }) + + const test = createTest('test one') + test.art = { + message: 'Element not found', + stack: 'Error', + } + + event.dispatcher.emit(event.test.before, test) + await recorder.promise() + + const step = { + name: 'see', + toString: () => 'I see test', + status: 'failed', + startTime: Date.now(), + endTime: Date.now() + 100, + } + + event.dispatcher.emit(event.step.after, step) + await recorder.promise() + + event.dispatcher.emit(event.test.failed, test) + await recorder.promise() + + expect(test.artifacts.aiTrace).to.be.ok + }) + + it('should ignore steps matching ignoreSteps pattern', async () => { + aiTrace({ + enabled: true, + output: testsDir, + ignoreSteps: [/^grab/], + }) + + const test = createTest('test one') + event.dispatcher.emit(event.test.before, test) + await recorder.promise() + + const step = { + name: 'grabText', + toString: () => 'I grab text', + status: 'success', + startTime: Date.now(), + endTime: Date.now() + 100, + } + event.dispatcher.emit(event.step.after, step) + await recorder.promise() + + expect(helperStub.saveScreenshot.called).to.be.false + }) + + it('should not save duplicate steps', async () => { + aiTrace({ + enabled: true, + output: testsDir, + }) + + const test = createTest('test one') + event.dispatcher.emit(event.test.before, test) + await recorder.promise() + + const step = { + name: 'see', + toString: () => 'I see test', + status: 'success', + startTime: Date.now(), + endTime: Date.now() + 100, + } + + event.dispatcher.emit(event.step.after, step) + await recorder.promise() + + event.dispatcher.emit(event.step.after, step) + await recorder.promise() + + expect(helperStub.saveScreenshot.calledOnce).to.be.true + }) + + it('should not create trace for BeforeSuite failures', async () => { + aiTrace({ + enabled: true, + output: testsDir, + }) + + const test = createTest('test one') + test.artifacts = {} + event.dispatcher.emit(event.test.before, test) + await recorder.promise() + + event.dispatcher.emit(event.test.failed, test, null, 'BeforeSuite') + await recorder.promise() + + expect(test.artifacts.aiTrace).to.be.undefined + }) + + it('should not create trace for AfterSuite failures', async () => { + aiTrace({ + enabled: true, + output: testsDir, + }) + + const test = createTest('test one') + test.artifacts = {} + event.dispatcher.emit(event.test.before, test) + await recorder.promise() + + event.dispatcher.emit(event.test.failed, test, null, 'AfterSuite') + await recorder.promise() + + expect(test.artifacts.aiTrace).to.be.undefined + }) + + describe('Artifact capture options', () => { + it('should not capture HTML when captureHTML is false', async () => { + aiTrace({ + enabled: true, + output: testsDir, + captureHTML: false, + }) + + const test = createTest('test one') + event.dispatcher.emit(event.test.before, test) + await recorder.promise() + + const step = { + name: 'amOnPage', + toString: () => 'I am on page', + status: 'success', + startTime: Date.now(), + endTime: Date.now() + 100, + } + + event.dispatcher.emit(event.step.after, step) + await recorder.promise() + + expect(helperStub.grabSource.called).to.be.false + }) + + it('should not capture ARIA when captureARIA is false', async () => { + aiTrace({ + enabled: true, + output: testsDir, + captureARIA: false, + }) + + const test = createTest('test one') + event.dispatcher.emit(event.test.before, test) + await recorder.promise() + + const step = { + name: 'amOnPage', + toString: () => 'I am on page', + status: 'success', + startTime: Date.now(), + endTime: Date.now() + 100, + } + + event.dispatcher.emit(event.step.after, step) + await recorder.promise() + + expect(helperStub.grabAriaSnapshot.called).to.be.false + }) + + it('should not capture browser logs when captureBrowserLogs is false', async () => { + aiTrace({ + enabled: true, + output: testsDir, + captureBrowserLogs: false, + }) + + const test = createTest('test one') + event.dispatcher.emit(event.test.before, test) + await recorder.promise() + + const step = { + name: 'amOnPage', + toString: () => 'I am on page', + status: 'success', + startTime: Date.now(), + endTime: Date.now() + 100, + } + + event.dispatcher.emit(event.step.after, step) + await recorder.promise() + + expect(helperStub.grabBrowserLogs.called).to.be.false + }) + }) + + describe('Failed step handling', () => { + it('should skip queued steps after test fails', async () => { + aiTrace({ + enabled: true, + output: testsDir, + }) + + event.dispatcher.emit(event.suite.before, {}) + await recorder.promise() + + const test = createTest('failing test') + test.art = { message: 'Test failed' } + + event.dispatcher.emit(event.test.before, test) + await recorder.promise() + + const step1 = { + name: 'see', + toString: () => 'I see success', + status: 'success', + startTime: Date.now(), + endTime: Date.now() + 100, + } + + const step2 = { + name: 'click', + toString: () => 'I click button', + status: 'failed', + startTime: Date.now() + 100, + endTime: Date.now() + 200, + } + + const step3 = { + name: 'fillField', + toString: () => 'I fill field', + status: 'queued', + startTime: Date.now() + 200, + endTime: Date.now() + 300, + } + + event.dispatcher.emit(event.step.after, step1) + await recorder.promise() + + event.dispatcher.emit(event.step.after, step2) + await recorder.promise() + + event.dispatcher.emit(event.step.after, step3) + await recorder.promise() + + event.dispatcher.emit(event.test.failed, test) + await recorder.promise() + + expect(helperStub.saveScreenshot.calledTwice).to.be.true + }) + }) + + describe('Step file naming', () => { + it('should use step names in file names instead of just numbers', async () => { + aiTrace({ + enabled: true, + output: testsDir, + }) + + event.dispatcher.emit(event.suite.before, {}) + await recorder.promise() + + const test = createTest('naming test') + event.dispatcher.emit(event.test.before, test) + await recorder.promise() + + const step = { + name: 'see', + toString: () => 'I see "Test Element"', + status: 'success', + startTime: Date.now(), + endTime: Date.now() + 100, + } + + event.dispatcher.emit(event.step.after, step) + await recorder.promise() + + expect(helperStub.saveScreenshot.calledOnce).to.be.true + + const screenshotPath = helperStub.saveScreenshot.firstCall.args[0] + expect(screenshotPath).to.include('I_see_Test_Element') + expect(screenshotPath).to.match(/_I_see_Test_Element_screenshot\.png$/) + }) + }) + + describe('Directory cleanup', () => { + it('should clean directory before each test run', async () => { + aiTrace({ + enabled: true, + output: testsDir, + }) + + event.dispatcher.emit(event.suite.before, {}) + await recorder.promise() + + const test = createTest('cleanup test') + event.dispatcher.emit(event.test.before, test) + await recorder.promise() + + const step = { + name: 'see', + toString: () => 'I see test', + status: 'success', + startTime: Date.now(), + endTime: Date.now() + 100, + } + + event.dispatcher.emit(event.step.after, step) + await recorder.promise() + + event.dispatcher.emit(event.test.passed, test) + await recorder.promise() + + const traceFile = test.artifacts.aiTrace + expect(traceFile).to.be.ok + expect(fs.existsSync(traceFile)).to.be.true + }) + }) +})