diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 47606314e..8adaa0996 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -1,51 +1,65 @@ name: Conformance Tests on: - push: - branches: [main] - pull_request: - workflow_dispatch: + push: + branches: [main] + pull_request: + workflow_dispatch: concurrency: - group: conformance-${{ github.ref }} - cancel-in-progress: true + group: conformance-${{ github.ref }} + cancel-in-progress: true permissions: - contents: read + contents: read jobs: - client-conformance: - runs-on: ubuntu-latest - continue-on-error: true - steps: - - uses: actions/checkout@v4 - - name: Install pnpm - uses: pnpm/action-setup@v4 - with: - run_install: false - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: pnpm - cache-dependency-path: pnpm-lock.yaml - - run: pnpm install - - run: pnpm run build:all - - run: pnpm run test:conformance:client:all + client-conformance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install pnpm + uses: pnpm/action-setup@v4 + with: + run_install: false + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + cache-dependency-path: pnpm-lock.yaml + - run: pnpm install + - run: pnpm run build:all + - uses: modelcontextprotocol/conformance@4a42c25058c4636aca1ae7f6547ee30805de36ee + with: + mode: client + command: "npx tsx src/conformance/everything-client.ts" + suite: all + expected-failures: ./conformance-baseline.yml + node-version: "24" - server-conformance: - runs-on: ubuntu-latest - continue-on-error: true - steps: - - uses: actions/checkout@v4 - - name: Install pnpm - uses: pnpm/action-setup@v4 - with: - run_install: false - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: pnpm - cache-dependency-path: pnpm-lock.yaml - - run: pnpm install - - run: pnpm run build:all - - run: pnpm run test:conformance:server + server-conformance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install pnpm + uses: pnpm/action-setup@v4 + with: + run_install: false + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + cache-dependency-path: pnpm-lock.yaml + - run: pnpm install + - run: pnpm run build:all + - name: Start everything-server + run: | + npx tsx src/conformance/everything-server.ts & + timeout 15 bash -c 'until curl -s http://localhost:3000/mcp > /dev/null 2>&1; do sleep 0.5; done' + echo "Server ready" + - uses: modelcontextprotocol/conformance@4a42c25058c4636aca1ae7f6547ee30805de36ee + with: + mode: server + url: http://localhost:3000/mcp + expected-failures: ./conformance-baseline.yml + node-version: "24" diff --git a/conformance-baseline.yml b/conformance-baseline.yml new file mode 100644 index 000000000..d67473894 --- /dev/null +++ b/conformance-baseline.yml @@ -0,0 +1,13 @@ +# MCP Conformance Test Expected Failures +# List scenario names that are known to fail in this SDK. +# Format: +# server: +# - scenario-name-1 +# - scenario-name-2 +# client: +# - scenario-name-3 +# +# The conformance runner will: +# - Exit 0 if only expected failures occur +# - Exit 1 if unexpected failures occur (regression) +# - Exit 1 if expected failures now pass (stale baseline, update this file)