diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index 989e36bad75..5cd77c772fa 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -94,7 +94,7 @@ jobs: - name: Install node dependencies if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} - run: yarn + run: npm install --loglevel=error --no-audit working-directory: ./tests/integration/playwright shell: bash @@ -105,7 +105,7 @@ jobs: - name: Install MECA validator if: ${{ runner.os != 'Windows' }} - run: npm install -g meca + run: npm install -g meca --loglevel=error --no-audit - name: Set RENV_PATHS_ROOT shell: bash diff --git a/tests/README.md b/tests/README.md index c3ea302d63c..b2fb336e796 100644 --- a/tests/README.md +++ b/tests/README.md @@ -35,6 +35,44 @@ Running tests require to have a local environment setup with Quarto development, To help with this configuration, the `tests/` folder contains `configure-test-env.sh` and `configure-test-env.ps1`. It will check for the tools and update the dependencies to what is used by Quarto tests. Running the script at least one will insure you are correctly setup. Then, it is run as part of running the tests so that dependencies are always updated. Set `QUARTO_TESTS_NO_CONFIG` to skip this step when running tests. +#### Optional test dependencies + +The configure scripts also check for optional tools that some tests require. Tests will gracefully skip when these tools are not available, but having them installed enables full test coverage: + +**Java** (version 8, 11, 17, or 21) + +- Required for: PDF standard validation tests using veraPDF +- The script will install veraPDF automatically if Java is found using `quarto install verapdf` + +**Node.js** (version 18 or later) and **npm** + +- Required for: Playwright integration tests and JATS/MECA validation +- Installation: Download from https://nodejs.org/ or use a version manager like nvm +- The script will: + - Check Node.js version and warn if < 18 + - Install the `meca` package globally for MECA validation + - Install Playwright and its dependencies + - Set up the multiplex server for Playwright tests + - Install Playwright browsers (Chrome, Firefox, etc.) + +**pdftotext** (from poppler) + +- Required for: Some PDF text extraction tests +- Installation: + - Ubuntu/Debian: `sudo apt-get install poppler-utils` + - macOS: `brew install poppler` + - Windows: `scoop install poppler` (auto-installed if Scoop is available) + +**rsvg-convert** (from librsvg) + +- Required for: PDF tests with SVG image conversion +- Installation: + - Ubuntu/Debian: `sudo apt-get install librsvg2-bin` + - macOS: `brew install librsvg` + - Windows: `scoop install librsvg` (auto-installed if Scoop is available) + +On Windows, the scripts will attempt to auto-install poppler and librsvg via Scoop if it's available on your system. + Dependencies are managed using the following tools: #### R @@ -102,20 +140,80 @@ Tests are run using `run-tests.sh` on UNIX, and `run-tests.ps1` on Windows. ./run-tests.ps1 smoke/extensions/extension-render-doc.test.ts ``` -#### Prevent configuration for dependencies (R, Julia, Python, ...) +#### Test environment variables + +The test scripts support several environment variables to control their behavior: -Those files will run `configure-test-env` scripts to check for requirements and set up dependencies (except on Github Action as this is done in the workflow file directly). -You can prevent test configuration locally by setting `QUARTO_TESTS_NO_CONFIG` environment variable to a non-empty value. +**QUARTO_TESTS_NO_CONFIG** +- Skip running `configure-test-env` scripts +- Useful for faster test runs when environment is already configured +- Tests will still activate `.venv` if present ```bash QUARTO_TESTS_NO_CONFIG="true" ./run-tests.sh ``` ```powershell -$env:QUARTO_TESTS_NO_CONFIG=$true +$env:QUARTO_TESTS_NO_CONFIG="true" +./run-tests.ps1 +``` + +**QUARTO_TESTS_FORCE_NO_VENV** (replaces deprecated `QUARTO_TESTS_FORCE_NO_PIPENV`) +- Skip activating the `.venv` virtual environment +- Tests will use system Python packages instead of UV-managed dependencies +- Use with caution: Python tests may fail if dependencies aren't in system Python + +```bash +QUARTO_TESTS_FORCE_NO_VENV="true" ./run-tests.sh +``` + +```powershell +$env:QUARTO_TESTS_FORCE_NO_VENV="true" ./run-tests.ps1 ``` +**Quick test runs with run-fast-tests scripts** + +For convenience, `run-fast-tests.sh` and `run-fast-tests.ps1` are provided to skip environment configuration: + +```bash +# Linux/macOS +./run-fast-tests.sh + +# Windows +./run-fast-tests.ps1 +``` + +These scripts set `QUARTO_TESTS_NO_CONFIG` automatically. Use after running `configure-test-env` at least once. + +**QUARTO_TEST_KEEP_OUTPUTS** (or use `--keep-outputs`/`-k` flag) +- Keep test output artifacts instead of cleaning them up +- Useful for debugging test failures or inspecting generated files +- Can be set via environment variable or command-line flag + +```bash +# Using flag +./run-tests.sh --keep-outputs +./run-tests.sh -k + +# Using environment variable +QUARTO_TEST_KEEP_OUTPUTS="true" ./run-tests.sh +``` + +```powershell +# Using flag +./run-tests.ps1 --keep-outputs +./run-tests.ps1 -k + +# Using environment variable +$env:QUARTO_TEST_KEEP_OUTPUTS="true" +./run-tests.ps1 +``` + +**Other environment variables** +- `QUARTO_TEST_VERBOSE` - Enable verbose test output +- `QUARTO_TESTS_NO_CHECK` - Not currently used (legacy variable) + #### About smoke-all tests `docs/smoke-all/` is a specific folder to run some tests written directly within `.qmd`, `.md` or `.ipynb` files (but files starting with `_` will be ignored). They are run through the `smoke/smoke-all.tests.ts` script. To ease running smoke-all tests, `run-tests.sh` has a special behavior where it will run `./smoke/smoke-all.tests.ts` when passed a `.qmd`, `.md` or `.ipynb` file, not starting with `_`. @@ -284,6 +382,7 @@ _quarto: The snapshot file should be saved alongside the output with a `.snapshot` extension (e.g., `output.html.snapshot`). When a snapshot test fails: + - A **unified diff** is displayed with colored output (red for removed, green for added) - A **word-level diff** shows changes with surrounding context - For **whitespace-only changes**, special markers visualize invisible characters: diff --git a/tests/configure-test-env.ps1 b/tests/configure-test-env.ps1 index 7b64252f843..c5ba764ce63 100644 --- a/tests/configure-test-env.ps1 +++ b/tests/configure-test-env.ps1 @@ -47,15 +47,68 @@ If ([string]::IsNullOrEmpty($env:GH_TOKEN)) { } quarto install tinytex -# Get npm in place +# Check for veraPDF (and Java) as the tool is required for PDF standard validation tests +Write-Host -ForegroundColor green ">>>> Checking Java for veraPDF" +try { $null = gcm java -ea stop; $java=$true } catch { + Write-Host -ForegroundColor red "No java found in PATH - veraPDF requires Java to be installed." + Write-Host -ForegroundColor red "Install Java and add to PATH if you need PDF standard validation tests." + Write-Host -ForegroundColor red "See: https://www.java.com/en/download/" +} + +If ($java) { + $javaVersion = java -version 2>&1 | Select-Object -First 1 + Write-Host "Java found: $javaVersion" +} + +Write-Host -ForegroundColor green ">>>> Installing veraPDF for PDF standard validation" +If ($java) { + quarto install verapdf +} Else { + Write-Host -ForegroundColor yellow "Skipping veraPDF installation (Java not found)" +} + +# Check Node.js and npm --- Write-Host -ForegroundColor green ">>>> Configuring npm for MECA testing environment" +try {$null = gcm node -ea stop; $node_exists=$true } catch { + Write-Host -ForegroundColor red "No node found - will skip any tests that require npm (e.g. JATS / MECA validation)" +} try {$null = gcm npm -ea stop; $npm_exists=$true } catch { - Write-Host -ForegroundColor red "No npm found - will skip any tests that require npm (e.g. JATS / MECA validation)" + Write-Host -ForegroundColor red "No npm found - npm is required but node is present" +} + +If ($node_exists) { + If ($npm_exists) { + # Check Node.js version + $nodeVersionFull = node -v + $nodeVersion = [int]($nodeVersionFull -replace 'v(\d+)\..*','$1') + Write-Host "Node.js version: $nodeVersionFull" + If ($nodeVersion -lt 18) { + Write-Host -ForegroundColor yellow "Warning: Node.js version $nodeVersion is older than recommended (18+)" + Write-Host -ForegroundColor yellow "Some tests may fail. Consider upgrading Node.js." + } + Write-Host "Setting up npm testing environment" + npm install -g meca --loglevel=error --no-audit + } } + +# Setup Playwright for browser testing --- +Write-Host -ForegroundColor green ">>>> Configuring Playwright for integration tests" If ($npm_exists) { - # TODO: Check to do equivalent of virtualenv - Write-Host "Setting up npm testing environment" - npm install -g meca + Write-Host "Installing Playwright dependencies..." + Push-Location integration/playwright + npm install --loglevel=error --no-audit + # Install multiplex server dependencies + Write-Host "Installing multiplex server dependencies..." + Push-Location multiplex-server + npm install --loglevel=error --no-audit + Pop-Location + # On Windows, npx playwright install --with-deps works without admin rights + Write-Host "Installing Playwright browsers..." + npx playwright install --with-deps + Pop-Location + Write-Host "Playwright browsers installed." +} Else { + Write-Host -ForegroundColor yellow "Skipping Playwright setup (npm not found)" } # Other tests dependencies @@ -66,7 +119,21 @@ try {$null = gcm pdftotext -ea stop; $pdftotext=$true } catch { Write-Host -ForegroundColor red "No scoop found - Scoop is a package manager for Windows - see https://scoop.sh/ and it can install poppler" } If($scoop) { - Write-Host -ForegroundColor green "Scoop is found so trying to install poppler for pdftotext" + Write-Host -ForegroundColor green "Scoop is found so trying to install poppler for pdftotext" scoop install poppler } +} + +# Check rsvg-convert for SVG conversion --- +Write-Host -ForegroundColor green ">>>> Checking rsvg-convert from librsvg" +try {$null = gcm rsvg-convert -ea stop; $rsvg=$true } catch { + Write-Host -ForegroundColor red "No rsvg-convert found - Some PDF tests with SVG images will be skipped." + Write-Host -ForegroundColor yellow "Install librsvg to enable SVG to PDF conversion tests:" + try {$null = gcm scoop -ea stop; $scoop=$true } catch {} + If($scoop) { + Write-Host -ForegroundColor green "Scoop is found, trying to install librsvg" + scoop install librsvg + } Else { + Write-Host -ForegroundColor red "Consider installing scoop (https://scoop.sh/) to easily install librsvg" + } } \ No newline at end of file diff --git a/tests/configure-test-env.sh b/tests/configure-test-env.sh index 17e53e0d24e..0c67327ed51 100755 --- a/tests/configure-test-env.sh +++ b/tests/configure-test-env.sh @@ -53,26 +53,95 @@ then export GH_TOKEN=$(gh auth token) fi -if [ -n $(command -v quarto) ] +if [ -n $(command -v quarto) ] then quarto install tinytex fi -# Get npm in place +# Check for veraPDF (and Java) as the tool is required for PDF standard validation tests +echo ">>>> Checking Java for veraPDF" +java_exists=$(command -v java) +if [ -z $java_exists ] +then + echo "No java found in PATH - veraPDF requires Java to be installed." + echo "Install Java and add to PATH if you need PDF standard validation tests." + echo "See: https://www.java.com/en/download/" +else + echo "Java found: $(java -version 2>&1 | head -n 1)" +fi + +# Install veraPDF for PDF standard validation --- +echo ">>>> Installing veraPDF for PDF standard validation" +if [ -n $(command -v quarto) ] +then + if [ -n "$java_exists" ] + then + quarto install verapdf + else + echo "Skipping veraPDF installation (Java not found)" + fi +else + echo "Skipping veraPDF installation (quarto not found)" +fi + +# Check Node.js and npm --- echo ">>>> Configuring npm for MECA testing environment" +node_exists=$(command -v node) npm_exists=$(command -v npm) -if [ -z $npm_exists ] +if [ -z $node_exists ] +then + echo "No node found - will skip any tests that require npm (e.g. JATS / MECA validation)" +elif [ -z $npm_exists ] then - echo "No npm found - will skip any tests that require npm (e.g. JATS / MECA validation)" + echo "No npm found - npm is required but node is present" else + # Check Node.js version + node_version=$(node -v | sed 's/v//' | cut -d. -f1) + echo "Node.js version: $(node -v)" + if [ "$node_version" -lt 18 ]; then + echo "Warning: Node.js version $node_version is older than recommended (18+)" + echo "Some tests may fail. Consider upgrading Node.js." + fi echo "Setting up npm testing environment" - npm install -g meca + npm install -g meca --loglevel=error --no-audit +fi + +# Setup Playwright for browser testing --- +echo ">>>> Configuring Playwright for integration tests" +if [ -n "$npm_exists" ] +then + echo "Installing Playwright dependencies..." + pushd integration/playwright > /dev/null + npm install --loglevel=error --no-audit + # Install multiplex server dependencies + echo "Installing multiplex server dependencies..." + pushd multiplex-server > /dev/null + npm install --loglevel=error --no-audit + popd > /dev/null + # Try to install browsers with --with-deps (may require sudo on Linux/macOS) + echo "Installing Playwright browsers..." + npx playwright install --with-deps || echo "Note: Browser installation may require sudo. Run manually: npx playwright install --with-deps" + popd > /dev/null +else + echo "Skipping Playwright setup (npm not found)" fi -# Get npm in place +# Check pdftotext --- echo ">>>> Do you have pdftotext installed (from poppler) ?" pdftotext_exists=$(command -v pdftotext) if [ -z $pdftotext_exists ] then echo "No pdftotext found - Some tests will require it, so you may want to install it." +fi + +# Check rsvg-convert for SVG conversion --- +echo ">>>> Do you have rsvg-convert installed (from librsvg) ?" +rsvg_exists=$(command -v rsvg-convert) +if [ -z $rsvg_exists ] +then + echo "No rsvg-convert found - Some PDF tests with SVG images will be skipped." + echo "Install librsvg to enable SVG to PDF conversion tests:" + echo " - Ubuntu/Debian: sudo apt-get install librsvg2-bin" + echo " - macOS: brew install librsvg" + echo " - Windows: scoop install librsvg" fi \ No newline at end of file diff --git a/tests/integration/playwright/yarn.lock b/tests/integration/playwright/yarn.lock deleted file mode 100644 index 84dd51f91de..00000000000 --- a/tests/integration/playwright/yarn.lock +++ /dev/null @@ -1,29 +0,0 @@ -# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. -# yarn lockfile v1 - - -"@playwright/test@^1.28.1": - version "1.47.0" - resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.47.0.tgz#69fc55b10754147cc20021afbfa05747d4961bf0" - integrity sha512-SgAdlSwYVpToI4e/IH19IHHWvoijAYH5hu2MWSXptRypLSnzj51PcGD+rsOXFayde4P9ZLi+loXVwArg6IUkCA== - dependencies: - playwright "1.47.0" - -fsevents@2.3.2: - version "2.3.2" - resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" - integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== - -playwright-core@1.47.0: - version "1.47.0" - resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.47.0.tgz#b54ec060fd83e5c2e46b63986b5ebb5e96ace427" - integrity sha512-1DyHT8OqkcfCkYUD9zzUTfg7EfTd+6a8MkD/NWOvjo0u/SCNd5YmY/lJwFvUZOxJbWNds+ei7ic2+R/cRz/PDg== - -playwright@1.47.0: - version "1.47.0" - resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.47.0.tgz#fb9b028883fad11362f9ff63ce7ba44bda0bf626" - integrity sha512-jOWiRq2pdNAX/mwLiwFYnPHpEZ4rM+fRSQpRHwEwZlP2PUANvL3+aJOF/bvISMhFD30rqMxUB4RJx9aQbfh4Ww== - dependencies: - playwright-core "1.47.0" - optionalDependencies: - fsevents "2.3.2" diff --git a/tests/run-fast-tests.ps1 b/tests/run-fast-tests.ps1 new file mode 100644 index 00000000000..c09dd519747 --- /dev/null +++ b/tests/run-fast-tests.ps1 @@ -0,0 +1,11 @@ +# Run tests without reconfiguring the environment (faster for repeated test runs) +# +# Environment variables set: +# QUARTO_TESTS_NO_CONFIG - Skip running configure-test-env.ps1 +# (don't reinstall/update R, Python, Julia dependencies) +# +# Note: This still activates .venv if present, so Python tests use correct dependencies. +# Only use after running configure-test-env.ps1 at least once to set up environment. + +$env:QUARTO_TESTS_NO_CONFIG="true" +& .\run-tests.ps1 @args diff --git a/tests/run-fast-tests.sh b/tests/run-fast-tests.sh index 1361bcaed9d..03237024e9e 100755 --- a/tests/run-fast-tests.sh +++ b/tests/run-fast-tests.sh @@ -1,3 +1,16 @@ #!/usr/bin/env bash -QUARTO_TESTS_FORCE_NO_PIPENV=true QUARTO_TESTS_NO_CONFIG=true QUARTO_TESTS_NO_CHECK=true ./run-tests.sh $* \ No newline at end of file +# Run tests without reconfiguring the environment (faster for repeated test runs) +# +# Environment variables set: +# QUARTO_TESTS_FORCE_NO_VENV - Skip activating .venv virtual environment +# (tests use system Python packages instead) +# QUARTO_TESTS_NO_CONFIG - Skip running configure-test-env.sh +# (don't reinstall/update R, Python, Julia dependencies) +# QUARTO_TESTS_NO_CHECK - Not currently used by run-tests.sh +# (kept for backward compatibility in case checked elsewhere) +# +# Note: Only use after running configure-test-env.sh at least once to set up environment. +# Consider removing QUARTO_TESTS_FORCE_NO_VENV if Python tests require .venv packages. + +QUARTO_TESTS_FORCE_NO_VENV=true QUARTO_TESTS_NO_CONFIG=true QUARTO_TESTS_NO_CHECK=true ./run-tests.sh $* \ No newline at end of file diff --git a/tests/run-tests.ps1 b/tests/run-tests.ps1 index d01ac6f6e6e..f0f3ef77f7c 100644 --- a/tests/run-tests.ps1 +++ b/tests/run-tests.ps1 @@ -80,31 +80,53 @@ if ( $MyInvocation.Line -eq "" ) { # when script is ran from a child process using -F # e.g pwsh -F ./run-tests.ps1 smoke/smoke-all.test.ts -- docs\smoke-all\2023\02\08\4272.qmd $customArgs = $MyInvocation.UnboundArguments +} elseif ($MyInvocation.InvocationName -eq '&') { + # when script is called via call operator from another script + # e.g & .\run-tests.ps1 @args + # Use UnboundArguments directly as it contains the actual arguments passed + $customArgs = $MyInvocation.UnboundArguments } elseif ($MyInvocation.Line -match "^[.] '[^']*'") { # when script is ran from a child process using -command # e.g pwsh -command ". 'run-tests.ps1' smoke/smoke-all.test.ts -- docs\smoke-all\2023\02\08\4272.qmd" # This is what happens on GHA when using 'run: |' and 'shell: pwsh' $argList = ($MyInvocation.Line -replace "^[.] '[^']*'\s*" -split '[;|]')[0].Trim() # Extract the argument list from the invocation command line. - + # Use Invoke-Expression with a Write-Output call to parse the raw argument list, # performing evaluation and splitting it into an array: - $customArgs = $argList ? @(Invoke-Expression "Write-Output -- $argList") : @() + $customArgs = $argList ? @(Invoke-Expression "Write-Output -- $argList") : @() } else { # When script is called from main process # e.g ./run-tests.ps1 smoke/smoke-all.test.ts -- docs\smoke-all\2023\02\08\4272.qmd $argList = ($MyInvocation.Line -replace ('^.*' + [regex]::Escape($MyInvocation.InvocationName)) -split '[;|]')[0].Trim() # Extract the argument list from the invocation command line. - + # Use Invoke-Expression with a Write-Output call to parse the raw argument list, # performing evaluation and splitting it into an array: - $customArgs = $argList ? @(Invoke-Expression "Write-Output -- $argList") : @() + $customArgs = $argList ? @(Invoke-Expression "Write-Output -- $argList") : @() +} + +# Check if keep-outputs mode is enabled and filter it from arguments +$KEEP_OUTPUTS = $false +$FILTERED_CUSTOM_ARGS = @() +foreach ($arg in $customArgs) { + if ($arg -eq "--keep-outputs" -or $arg -eq "-k") { + $KEEP_OUTPUTS = $true + } else { + $FILTERED_CUSTOM_ARGS += $arg + } +} +$customArgs = $FILTERED_CUSTOM_ARGS + +if ($KEEP_OUTPUTS) { + $env:QUARTO_TEST_KEEP_OUTPUTS = "true" + Write-Host "> Keep outputs mode enabled - test artifacts will not be deleted" } ## Short version syntax to run smoke-all.test.ts ## Only use if different than ./run-test.ps1 ./smoke/smoke-all.test.ts If ($customArgs[0] -notlike "*smoke-all.test.ts") { - + $SMOKE_ALL_TEST_FILE="./smoke/smoke-all.test.ts" # Check file argument $SMOKE_ALL_FILES=@() @@ -112,7 +134,8 @@ If ($customArgs[0] -notlike "*smoke-all.test.ts") { ForEach ($file in $customArgs) { $filename=$(Split-Path -Path $file -Leaf) - If ($filename -match "^^[^_].*[.]qmd$" -Or $filename -match "^[^_].*[.]ipynb$" -Or $filename -match "^[^_].*[.]md$") { + + If ($filename -match "^[^_].*[.]qmd$" -Or $filename -match "^[^_].*[.]ipynb$" -Or $filename -match "^[^_].*[.]md$") { $SMOKE_ALL_FILES+=$file } elseif ($file -Like "*.ts") { $TESTS_TO_RUN+=$file