#!/usr/bin/env node
/*
 * check-optout-links.mjs
 *
 * Reads a removal tracker written in Markdown, pulls out every opt-out URL in it,
 * and tells you which ones still work. Run it monthly. Opt-out pages move, and a
 * tracker full of dead links is worse than no tracker, because it looks finished.
 *
 * It is deliberately read-only. It performs GET requests and nothing else. It never
 * submits a form, never sends your personal information anywhere, and never solves
 * a challenge. Removing yourself is still something you do yourself.
 *
 * No dependencies. Node 18 or later.
 *
 * Usage:
 *   node check-optout-links.mjs data-broker-removal-tracker.md
 *   node check-optout-links.mjs tracker.md --json > status.json
 *   node check-optout-links.mjs tracker.md --concurrency 6 --timeout 30000
 *
 * Exit code is 1 if anything is unreachable, so you can wire it into a scheduled job.
 *
 * From https://jwatte.com/blog/delete-yourself-from-data-brokers/
 * Free to copy and adapt. No warranty. Check what it does before you run it.
 */

import fs from 'node:fs';

const args = process.argv.slice(2);
const file = args.find((a) => !a.startsWith('--'));
const asJson = args.includes('--json');
const num = (flag, dflt) => {
  const i = args.indexOf(flag);
  return i === -1 ? dflt : Number(args[i + 1]) || dflt;
};
const CONCURRENCY = num('--concurrency', 8);
const TIMEOUT = num('--timeout', 20000);

if (!file) {
  console.error('usage: node check-optout-links.mjs <tracker.md> [--json] [--concurrency N] [--timeout MS]');
  process.exit(2);
}

const UA =
  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36';

const text = fs.readFileSync(file, 'utf8');

// Pull URLs out of markdown links, bare URLs and table cells alike.
const found = new Map();
for (const m of text.matchAll(/https?:\/\/[^\s)|<>"'`\]]+/g)) {
  let u = m[0].replace(/[.,;:]+$/, '');
  // Skip the article and site links, we only care about opt-out destinations.
  if (/jwatte\.com/i.test(u)) continue;
  if (!found.has(u)) {
    // Grab a little context so the report says which broker the link belongs to.
    const at = m.index ?? 0;
    const lineStart = text.lastIndexOf('\n', at) + 1;
    const line = text.slice(lineStart, text.indexOf('\n', at) === -1 ? undefined : text.indexOf('\n', at));
    const label = (line.match(/\|\s*([^|]{2,60}?)\s*\|/) || [, ''])[1].trim();
    found.set(u, label);
  }
}

const jobs = [...found.entries()].map(([url, label]) => ({ url, label }));
if (!jobs.length) {
  console.error(`no URLs found in ${file}`);
  process.exit(2);
}
if (!asJson) console.error(`checking ${jobs.length} URLs from ${file}\n`);

async function check(job) {
  const ctl = new AbortController();
  const timer = setTimeout(() => ctl.abort(), TIMEOUT);
  const started = Date.now();
  try {
    const r = await fetch(job.url, {
      method: 'GET',
      redirect: 'follow',
      signal: ctl.signal,
      headers: { 'user-agent': UA, accept: 'text/html,application/xhtml+xml,*/*' },
    });
    clearTimeout(timer);
    return { ...job, status: r.status, finalUrl: r.url, ms: Date.now() - started };
  } catch (e) {
    clearTimeout(timer);
    const code = String((e && (e.cause?.code || e.name)) || 'ERR');
    return { ...job, status: 'ERR', code, ms: Date.now() - started };
  }
}

// Classify honestly. A 403 from one of these sites usually means the page is fine
// and it is refusing anything that is not a human in a browser. That is not the
// same as a dead link, and reporting it as one would send you chasing nothing.
function verdict(r) {
  if (r.status === 200) return 'OK';
  if (r.status === 'ERR' && /ENOTFOUND|EAI_AGAIN/.test(r.code || '')) return 'DEAD';
  if (r.status === 404 || r.status === 410) return 'DEAD';
  if (r.status === 403 || r.status === 429 || r.status === 503) return 'BLOCKED';
  if (r.status === 'ERR') return 'ERROR';
  if (typeof r.status === 'number' && r.status >= 500) return 'SERVER';
  if (typeof r.status === 'number' && r.status >= 300) return 'MOVED';
  return 'OTHER';
}

const results = [];
let i = 0;
await Promise.all(
  Array.from({ length: Math.max(1, CONCURRENCY) }, async () => {
    while (i < jobs.length) {
      const job = jobs[i++];
      const r = await check(job);
      r.verdict = verdict(r);
      results.push(r);
      if (!asJson) {
        const mark = { OK: 'ok  ', DEAD: 'DEAD', BLOCKED: 'bot?', ERROR: 'err ', SERVER: '5xx ', MOVED: 'move', OTHER: '??  ' }[r.verdict];
        console.error(`  ${mark} ${String(r.status).padEnd(5)} ${r.url.slice(0, 78)}`);
      }
    }
  })
);

results.sort((a, b) => a.url.localeCompare(b.url));

if (asJson) {
  console.log(JSON.stringify({ file, checked: results.length, results }, null, 2));
} else {
  const by = (v) => results.filter((r) => r.verdict === v);
  console.log('\n=================== SUMMARY ===================');
  console.log(`  checked            ${results.length}`);
  console.log(`  reachable          ${by('OK').length}`);
  console.log(`  blocking automation ${by('BLOCKED').length}   (page is probably fine, open it yourself)`);
  console.log(`  dead               ${by('DEAD').length}   (fix the tracker)`);
  console.log(`  errored            ${by('ERROR').length + by('SERVER').length}`);

  const dead = [...by('DEAD'), ...by('ERROR'), ...by('SERVER')];
  if (dead.length) {
    console.log('\n  NEEDS ATTENTION');
    for (const d of dead) console.log(`    ${String(d.status).padEnd(5)} ${d.label ? d.label.slice(0, 30).padEnd(30) : ''} ${d.url}`);
  }
  const blocked = by('BLOCKED');
  if (blocked.length) {
    console.log('\n  OPEN THESE BY HAND, they refuse automated clients');
    for (const d of blocked) console.log(`    ${String(d.status).padEnd(5)} ${d.label ? d.label.slice(0, 30).padEnd(30) : ''} ${d.url}`);
  }
  console.log('');
}

process.exit(results.some((r) => ['DEAD', 'ERROR', 'SERVER'].includes(r.verdict)) ? 1 : 0);