AI Crawler Detection and Verification Levels

async function getBots(): Promise<Bot[]> {
  if (botsetCache && Date.now() - botsetCache.at < 3_600_000) return botsetCache.bots;
  try {
    const r = await fetch(`${AEN_BASE}/api/v1/edge/botset`, { headers: auth() });
    const j = await r.json();
    const bots: Bot[] = j?.botset?.bots ?? [];
    botsetCache = { at: Date.now(), bots };
    return bots;
  } catch {
    return botsetCache?.bots ?? [];
  }
}

function ipInCidrs(ip: string, cidrs: string[] = []): boolean {
  for (const c of cidrs) {
    try {
      const [net, bitsRaw] = c.split("/");
      const bits = Number(bitsRaw);
      if (ip.includes(":") || net.includes(":")) continue; // IPv4 only in this drop-in
      const a = ipToInt(ip), b = ipToInt(net);
      if (a === null || b === null) continue;
      const mask = bits === 0 ? 0 : (~0 << (32 - bits)) >>> 0;
      if ((a & mask) === (b & mask)) return true;
    } catch { /* skip */ }
  }
  return false;
}

function matchBot(ua: string, bots: Bot[]): Bot | null {
  const low = ua.toLowerCase();
  let best: Bot | null = null;
  for (const b of bots) {
    const t = b.uaToken.toLowerCase();
    if (new RegExp(`(?:^|[^a-z0-9])${escapeRe(t)}(?:[^a-z0-9]|$)`, "i").test(low)) {
      if (!best || b.uaToken.length > best.uaToken.length) best = b;
    }
  }
  return best;
}

const ENABLED = new Set(["USER_TRIGGERED", "RETRIEVAL"]);

// CLOAKING FIREWALL: classic search bots (Googlebot/Bingbot/Applebot) → always the human page.
if (bot.isSearchIndex) return NextResponse.next();