#!/usr/bin/env node /* * mcp-connector-scenario-tests.mjs * * Test a published MCP connector the way assistants and people actually use it, * not just the way the protocol says they should. * * A server can pass `initialize` and `tools/list` and still fail real users. When we * tested ours, 25 tools on 5 servers rejected a call because an assistant sent `null` * for an optional argument it left unset, and a blog connector answered "latest * articles" with a May post. This script checks for those problems and the others * below. * * What it checks, on one Streamable HTTP endpoint: * 1. Contract: initialize (with the initialized notification and any session ID the * server issues), tools/list, tool names, description length (1 to 1,024 * characters), inputSchema, and the readOnlyHint annotation. * 2. Compatibility, for every tool marked readOnlyHint: true: * - a plain call with sample arguments * - the same call with params._meta (must give the same outcome as the plain call) * - every optional argument set to null (must behave as if absent) * - every optional string set to "" (warning only) * - a wrong-typed argument (should fail cleanly: isError or -32602, never HTTP 5xx) * plus a call to a tool that does not exist. * 3. Freshness (optional, --feed): the newest items in your own RSS, Atom or JSON * feed must come back from your connector. With --recent-tool, a date-sorted * listing tool must return them newest first. With --search-tool, each item * must be findable by its own title. * 4. Links: every URL a tool hands back must open (HTTP status below 400). A 403, * 429 or 503 from another site is a warning, since many sites block scripts. * * Safety: read-only. A tool whose readOnlyHint is not exactly true is never called, * including the tools named in --recent-tool and --search-tool. Requests are paced at * about one per second per host. No credentials are sent. Point it only at servers you * run or have permission to test. * * Usage: * node mcp-connector-scenario-tests.mjs --endpoint https://example.com/mcp * node mcp-connector-scenario-tests.mjs --endpoint https://example.com/mcp \ * --feed https://example.com/feed.xml --recent-tool list_recent_articles \ * --search-tool search_site --n 5 * Options: --links 40 (cap on links checked), --json (print JSON), * --sample tool_name='{"slug":"real-value"}' (real sample arguments for a * tool whose required inputs a placeholder cannot satisfy; repeatable) * * Exit code: 0 all pass (warnings allowed), 1 any FAIL, 2 the script itself failed * (bad options, network setup, or an unexpected error). * Requires Node 18 or later. No dependencies. * * Companion to https://jwatte.com/blog/test-mcp-connector-like-real-users/ * Version 1.2, 2026-11-22. */ const PROTOCOL = '2025-06-18'; const UA = 'mcp-connector-scenario-tests/1.2 (+read-only)'; const argv = process.argv.slice(2); const usage = 'Usage: node mcp-connector-scenario-tests.mjs --endpoint https://example.com/mcp [--feed URL] [--recent-tool NAME] [--search-tool NAME] [--n 5] [--links 40] [--sample tool=JSON] [--json]'; function die(msg) { console.error(msg); console.error(usage); process.exit(2); } const opt = (k, d) => { const i = argv.indexOf(`--${k}`); return i >= 0 ? argv[i + 1] : d; }; const EP = opt('endpoint'); const FEED = opt('feed'); const RECENT = opt('recent-tool'); const SEARCH = opt('search-tool'); const N = Number(opt('n', '5')); const LINK_CAP = Number(opt('links', '40')); const JSON_OUT = argv.includes('--json'); const SAMPLES = {}; argv.forEach((a, i) => { if (a !== '--sample') return; const raw = argv[i + 1] || ''; const eq = raw.indexOf('='); if (eq < 1) die(`--sample needs tool_name='{"arg":"value"}', got: ${raw}`); try { SAMPLES[raw.slice(0, eq)] = JSON.parse(raw.slice(eq + 1)); } catch { die(`--sample value for ${raw.slice(0, eq)} is not valid JSON`); } }); if (!EP) die('Missing --endpoint.'); try { new URL(EP); if (FEED) new URL(FEED); } catch { die('--endpoint and --feed must be full URLs.'); } if (!(N > 0) || !(LINK_CAP >= 0)) die('--n must be above 0 and --links 0 or more.'); // ---------- pacing and transport ---------- const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); const chains = new Map(), lastHit = new Map(); function pace(url) { const host = new URL(url).host; const next = (chains.get(host) || Promise.resolve()).then(async () => { const wait = 1100 - (Date.now() - (lastHit.get(host) || 0)); if (wait > 0) await sleep(wait); lastHit.set(host, Date.now()); }); chains.set(host, next); return next; } let rpcId = 1; let sessionId = null; let negotiated = PROTOCOL; async function post(message) { await pace(EP); const headers = { 'content-type': 'application/json', accept: 'application/json, text/event-stream', 'user-agent': UA, 'mcp-protocol-version': negotiated }; if (sessionId) headers['mcp-session-id'] = sessionId; const t0 = Date.now(); try { const r = await fetch(EP, { method: 'POST', signal: AbortSignal.timeout(30000), headers, body: JSON.stringify(message) }); const sid = r.headers.get('mcp-session-id'); if (sid) sessionId = sid; return { status: r.status, text: await r.text(), ms: Date.now() - t0, type: r.headers.get('content-type') || '' }; } catch (e) { return { status: 0, text: '', ms: Date.now() - t0, err: e.message }; } } // A Streamable HTTP response is either one JSON body or an event stream that may carry // notifications before the response. Pick the message whose id matches the request. function pickResponse(text, id) { const candidates = []; const events = text.split(/\r?\n\r?\n/); for (const ev of events) { const data = ev.split(/\r?\n/).filter((l) => l.startsWith('data:')).map((l) => l.slice(5).trimStart()).join('\n'); if (data) candidates.push(data); } if (!candidates.length) candidates.push(text); for (const c of candidates) { try { const j = JSON.parse(c); if (j && j.id === id) return j; } catch {} } return null; } async function rpc(method, params) { const id = rpcId++; const r = await post({ jsonrpc: '2.0', id, method, params }); return { ...r, json: pickResponse(r.text, id) }; } async function call(name, args, meta) { const params = { name, arguments: args }; if (meta) params._meta = meta; const r = await rpc('tools/call', params); const res = r.json?.result; const out = res?.content?.filter((c) => c.type === 'text').map((c) => c.text).join('\n') || ''; return { ...r, rpcError: r.json?.error, isError: !!res?.isError, out }; } async function getUrl(url) { await pace(url); try { const r = await fetch(url, { redirect: 'follow', signal: AbortSignal.timeout(20000), headers: { 'user-agent': UA } }); return { status: r.status, text: await r.text() }; } catch (e) { return { status: 0, err: e.message, text: '' }; } } // ---------- results ---------- const results = []; function rec(id, status, detail = '') { results.push({ id, status, detail: String(detail).slice(0, 400) }); if (!JSON_OUT && status !== 'PASS') console.log(`${status.padEnd(4)} ${id}${detail ? ' :: ' + String(detail).slice(0, 200) : ''}`); } const ok = (id, cond, detail, level = 'FAIL') => rec(id, cond ? 'PASS' : level, cond ? '' : detail); const links = new Set(); const harvest = (text) => (text.match(/https?:\/\/[^\s)"'<>\]]+/g) || []).forEach((u) => links.add(u.replace(/[.,;:]+$/, ''))); const todayPlus = (n) => new Date(Date.now() + n * 864e5).toISOString().slice(0, 10); const NAMED = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', ndash: '-', mdash: '-', rsquo: "'", lsquo: "'", rdquo: '"', ldquo: '"', hellip: '...' }; const safeCp = (n, fallback) => { try { return String.fromCodePoint(n); } catch { return fallback; } }; function decode(s) { return String(s) .replace(/&#x([0-9a-f]+);/gi, (m, h) => safeCp(parseInt(h, 16), m)) .replace(/&#(\d+);/g, (m, d) => safeCp(Number(d), m)) .replace(/&([a-z]+);/gi, (m, n) => NAMED[n.toLowerCase()] ?? m); } // Compare titles loosely: decode entities, fold curly quotes and dashes, collapse space. const norm = (s) => decode(s).toLowerCase().replace(/[\u2018\u2019]/g, "'").replace(/[\u201c\u201d]/g, '"') .replace(/[\u2013\u2014]/g, '-').replace(/\s+/g, ' ').trim(); const props = (t) => (t.inputSchema && typeof t.inputSchema === 'object' && t.inputSchema.properties) || {}; const required = (t) => (t.inputSchema && Array.isArray(t.inputSchema.required) && t.inputSchema.required) || []; // Build arguments for a tool's required inputs when no sample was given. function sampleArgs(tool) { if (SAMPLES[tool.name]) return { ...SAMPLES[tool.name] }; const args = {}; for (const k of required(tool)) { const p = props(tool)[k] || {}; if (p.enum) args[k] = p.enum[0]; else if (p.default !== undefined) args[k] = p.default; else if (p.type === 'integer' || p.type === 'number') args[k] = p.minimum ?? 1; else if (/YYYY-MM-DD|date/i.test(`${p.description || ''}${p.format || ''}`)) args[k] = todayPlus(/checkout|end|depart/i.test(k) ? 21 : 14); else args[k] = 'test'; } return args; } const callOk = (r) => r.status === 200 && !!r.json && !r.rpcError && !r.isError; // The kind of answer a call produced, so a variant call can be compared with the plain one. const outcome = (r) => (!r.json ? 'no-response' : r.rpcError ? `rpc ${r.rpcError.code}` : r.isError ? 'isError' : 'ok'); // ---------- 1 and 2: contract and compatibility ---------- async function contract() { const init = await rpc('initialize', { protocolVersion: PROTOCOL, capabilities: {}, clientInfo: { name: 'mcp-connector-scenario-tests', version: '1.2' } }); const info = init.json?.result?.serverInfo; ok('initialize', init.status === 200 && info?.name, `HTTP ${init.status} ${init.err || init.text.slice(0, 120)}`); if (!info) return []; if (init.json.result.protocolVersion) negotiated = init.json.result.protocolVersion; ok('initialize returns instructions', !!init.json.result.instructions, 'no server instructions', 'WARN'); const note = await post({ jsonrpc: '2.0', method: 'notifications/initialized' }); ok('initialized notification accepted', note.status >= 200 && note.status < 300, `HTTP ${note.status}`, 'WARN'); const list = await rpc('tools/list', {}); const tools = list.json?.result?.tools || []; ok('tools/list returns tools', tools.length > 0, `HTTP ${list.status} ${list.text.slice(0, 120)}`); for (const t of tools) { ok(`${t.name}: name format`, /^[a-zA-Z0-9_-]{1,64}$/.test(t.name), t.name, 'WARN'); const len = (t.description || '').length; ok(`${t.name}: description 1 to 1,024 characters`, len >= 1 && len <= 1024, `length ${len}`, 'WARN'); ok(`${t.name}: inputSchema is an object`, t.inputSchema?.type === 'object', JSON.stringify(t.inputSchema ?? null).slice(0, 100)); ok(`${t.name}: readOnlyHint set`, typeof t.annotations?.readOnlyHint === 'boolean', 'annotations.readOnlyHint missing', 'WARN'); } for (const t of tools.filter((x) => x.annotations?.readOnlyHint === true)) { const args = sampleArgs(t); const plain = await call(t.name, args); ok(`${t.name}: plain call`, callOk(plain) && plain.out.length > 0, `HTTP ${plain.status} ${plain.rpcError?.message || plain.out.slice(0, 160)} (if it needs real inputs, pass --sample)`); ok(`${t.name}: answers within 8 s`, plain.ms < 8000, `${plain.ms} ms`, 'WARN'); harvest(plain.out); const meta = await call(t.name, args, { progressToken: 'scenario-test' }); ok(`${t.name}: accepts params._meta`, meta.status === 200 && outcome(meta) === outcome(plain), `plain call: ${outcome(plain)}; with _meta: ${outcome(meta)} ${meta.rpcError?.message || ''}`); const optional = Object.keys(props(t)).filter((k) => !required(t).includes(k)); if (optional.length) { const nulls = { ...args }; optional.forEach((k) => { nulls[k] = null; }); const r = await call(t.name, nulls); ok(`${t.name}: null optional arguments treated as absent`, r.status === 200 && outcome(r) === outcome(plain), `${optional.join(', ')} -> ${outcome(r)} ${r.rpcError?.message || r.out.slice(0, 140)} (plain call: ${outcome(plain)})`); const empties = { ...args }; optional.forEach((k) => { const p = props(t)[k]; if (p.type === 'string' && !p.enum) empties[k] = ''; }); if (JSON.stringify(empties) !== JSON.stringify(args)) { const e = await call(t.name, empties); ok(`${t.name}: empty-string optional arguments handled`, e.status === 200 && !!e.json && !e.rpcError, `HTTP ${e.status} ${e.rpcError?.message || ''}`, 'WARN'); } } const first = Object.entries(props(t))[0]; if (first) { const bad = { ...args, [first[0]]: first[1].type === 'string' ? 12345 : 'not-a-number' }; const r = await call(t.name, bad); if (r.status >= 500 || r.status === 0 || (!r.json && r.status !== 200)) rec(`${t.name}: wrong-type argument fails cleanly`, 'FAIL', `HTTP ${r.status} ${r.text.slice(0, 120)}`); else if (r.isError || r.rpcError?.code === -32602) rec(`${t.name}: wrong-type argument fails cleanly`, 'PASS'); else if (r.rpcError) rec(`${t.name}: wrong-type argument fails cleanly`, 'WARN', `error code ${r.rpcError.code}, expected -32602 or isError`); else rec(`${t.name}: wrong-type argument fails cleanly`, 'WARN', 'the server accepted a wrong-typed value'); } } const unknown = await call('no_such_tool_scenario_test', {}); ok('unknown tool fails cleanly', unknown.status > 0 && unknown.status < 500 && (unknown.rpcError || unknown.isError), `HTTP ${unknown.status}`); return tools; } // ---------- 3: freshness against your own feed ---------- function parseFeed(text) { try { // JSON Feed const j = JSON.parse(text); if (Array.isArray(j.items)) return j.items.map((i) => ({ title: i.title || '', url: i.url || i.id || '', date: i.date_published || i.date_modified || '' })); } catch {} return [...text.matchAll(/<(item|entry)\b[\s\S]*?<\/\1>/g)].map((m) => { const b = m[0]; const pick = (re) => (b.match(re) || [])[1] || ''; const alt = [...b.matchAll(/]*)\/?>/g)].map((x) => x[1]).find((attrs) => /href=/.test(attrs) && (!/rel=/.test(attrs) || /rel="alternate"/.test(attrs))); return { title: decode(pick(/]*>(?:)?<\/title>/)), url: pick(/([^<]+)<\/link>/) || (alt ? (alt.match(/href="([^"]+)"/) || [])[1] : ''), date: pick(/([^<]+)([^<]+)([^<]+) x.name === name); if (!t) { rec(`${label} ${name} exists`, 'FAIL', 'not in tools/list'); return null; } if (t.annotations?.readOnlyHint !== true) { rec(`${label} ${name} is read-only`, 'FAIL', 'readOnlyHint is not true, so it was not called'); return null; } return t; } async function freshness(tools) { const feed = await getUrl(FEED); ok(`feed opens ${FEED}`, feed.status === 200, `HTTP ${feed.status} ${feed.err || ''}`); const items = parseFeed(feed.text).filter((i) => i.title) .sort((a, b) => (Date.parse(b.date) || 0) - (Date.parse(a.date) || 0)).slice(0, N); ok(`feed has ${N} dated items`, items.length === N && items.every((i) => Date.parse(i.date)), `found ${items.length}`, 'WARN'); if (!items.length) return; const recentTool = RECENT ? readOnlyTool(tools, RECENT, 'recent tool') : null; if (recentTool) { const hasLimit = 'limit' in props(recentTool); const r = await call(RECENT, hasLimit ? { limit: N } : {}); harvest(r.out); const text = norm(r.out); const pos = items.map((i) => text.indexOf(norm(i.title))); ok(`${RECENT}: returns the ${N} newest feed items`, pos.every((p) => p >= 0), `missing: ${items.filter((_, k) => pos[k] < 0).map((i) => i.title).join(' | ')}`); // Newest first: an item with a strictly later date must appear earlier. Same-date ties may come in any order. let ordered = true; for (let a = 0; a < items.length; a++) for (let b = 0; b < items.length; b++) { if (pos[a] < 0 || pos[b] < 0) continue; if (Date.parse(items[a].date) > Date.parse(items[b].date) && pos[a] > pos[b]) ordered = false; } ok(`${RECENT}: newest first`, ordered, 'a newer item appears after an older one'); if (hasLimit) { const nulls = await call(RECENT, { limit: null }); ok(`${RECENT}: limit null treated as absent`, callOk(nulls), nulls.rpcError?.message || nulls.out.slice(0, 120)); } } if (SEARCH) { const t = readOnlyTool(tools, SEARCH, 'search tool'); if (!t) return; const key = required(t)[0] || Object.keys(props(t))[0] || 'query'; for (const i of items) { const r = await call(SEARCH, { [key]: i.title }); harvest(r.out); ok(`${SEARCH}: finds "${i.title.slice(0, 60)}" by its title`, norm(r.out).includes(norm(i.title)) || (i.url && r.out.includes(i.url)), r.out.slice(0, 120)); } // A recency word alone should be answered by date, not by keyword relevance. const latest = await call(SEARCH, { [key]: 'latest' }); ok(`${SEARCH}: "latest" includes the newest feed item`, norm(latest.out).includes(norm(items[0].title)), `newest is "${items[0].title.slice(0, 60)}"; answer starts: ${latest.out.slice(0, 120)}`, 'WARN'); } } // ---------- run ---------- try { const tools = await contract(); if (FEED && tools.length) await freshness(tools); const endpointHost = new URL(EP).host; // MCP endpoints usually reject a plain GET (405) or open an event stream, so skip them. const toCheck = [...links].filter((u) => u.replace(/\/$/, '') !== EP.replace(/\/$/, '') && !/\/mcp(-[a-z]+)?\/?$/.test(u)).slice(0, LINK_CAP); for (const u of toCheck) { const r = await getUrl(u); const own = (() => { try { return new URL(u).host === endpointHost; } catch { return false; } })(); const blocked = [403, 429, 503].includes(r.status); ok(`link opens ${u.slice(0, 90)}`, r.status > 0 && r.status < 400, `HTTP ${r.status} ${r.err || ''}`, !own && blocked ? 'WARN' : 'FAIL'); } } catch (e) { console.error(e); process.exit(2); } const count = (s) => results.filter((r) => r.status === s).length; const summary = { endpoint: EP, finished: new Date().toISOString(), pass: count('PASS'), warn: count('WARN'), fail: count('FAIL') }; if (JSON_OUT) console.log(JSON.stringify({ summary, results }, null, 2)); else console.log(`\nPASS ${summary.pass} WARN ${summary.warn} FAIL ${summary.fail} (${EP})`); process.exit(summary.fail ? 1 : 0);