Integrate any stack (Node, PHP, Ruby, nginx, Apache)

Not on Vercel, Cloudflare or WordPress? The Agent Edge Network is just four HTTP endpoints and a small piece of request-handling logic you drop into your own stack. This guide gives you a copy-paste snippet for Node.js, PHP and Ruby, plus how to route crawler traffic into it from nginx or Apache.

Every official drop-in — the Next.js middleware, Cloudflare Worker and WordPress plugin — is a thin wrapper around exactly the logic below. If your language isn’t listed, port the ~40 lines of pseudocode; it’s deliberately simple.

How it works

On each request you make a fast local decision, then either serve the artifact or pass through. You always fail open — if anything errors, serve your normal page.

Match the User-Agent against the verified bot set. Not a known crawler → pass through.
Cloaking firewall. If the bot is a search index (Googlebot/Bingbot/Applebot) → always pass through. See Cloaking firewall.
Verify the IP against the crawler’s published CIDR ranges → IP_VERIFIED vs UA_ONLY.
Check eligibility — verified, the purpose is enabled, and your serve mode is Live (default).
Serve the pre-generated artifact for sha256(url); otherwise pass through.
Report the hit to telemetry (fire-and-forget).

The endpoints

All calls send Authorization: Bearer <your edge token> (from Agent Analytics → Setup).

Endpoint	Method	Purpose
`/api/v1/edge/botset`	GET	Verified crawlers: `uaToken`, `cidrs`, `purpose`, `isSearchIndex`. Cache ~1h.
`/api/v1/edge/config`	GET	`serveMode` (`off`/`shadow`/`canary:<pct>`/`default`) + `enabledPurposes`. Cache ~5m.
`/api/v1/edge/artifact?u=<sha256(url)>`	GET	The `{ markdown, jsonLd, … }` for one URL.
`/api/v1/edge/telemetry`	POST	`{ hits: [ … ] }` — record what was crawled and served.

Base URL: https://app.trybluesky.com.

Snippets

Each snippet ships in shadow mode (log-only) — flip SERVE_MODE to "default" when you’re ready to serve. Set your edge token as the AEN_EDGE_TOKEN environment variable.

Node.js (Express)
PHP (any framework)
Ruby (Rack)

aen.js

// aen.js — Bluesky Agent Edge Network middleware for Express. Zero deps (Node 18+ fetch).
import crypto from "node:crypto";

const AEN_BASE = "https://app.trybluesky.com";
const TOKEN = process.env.AEN_EDGE_TOKEN;
const SERVE_MODE = "shadow"; // "shadow" | "default"
const ENABLED = new Set(["USER_TRIGGERED", "RETRIEVAL"]);

let botsetCache = null;
async function getBots() {
  if (botsetCache && Date.now() - botsetCache.at < 3_600_000) return botsetCache.bots;
  try {
    const r = await fetch(`${AEN_BASE}/api/v1/edge/botset`, { headers: { authorization: `Bearer ${TOKEN}` } });
    const j = await r.json();
    botsetCache = { at: Date.now(), bots: j?.botset?.bots ?? [] };
    return botsetCache.bots;
  } catch { return botsetCache?.bots ?? []; }
}

function matchBot(ua, bots) {
  const low = ua.toLowerCase();
  let best = null;
  for (const b of bots) {
    const t = b.uaToken.toLowerCase().replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
    if (new RegExp(`(?:^|[^a-z0-9])${t}(?:[^a-z0-9]|$)`, "i").test(low)) {
      if (!best || b.uaToken.length > best.uaToken.length) best = b;
    }
  }
  return best;
}
function ipToInt(ip) {
  const p = ip.split("."); if (p.length !== 4) return null;
  let n = 0; for (const o of p) { const v = Number(o); if (!Number.isInteger(v) || v < 0 || v > 255) return null; n = (n << 8) | v; }
  return n >>> 0;
}
function ipInCidrs(ip, cidrs = []) {
  if (ip.includes(":")) return false;
  const a = ipToInt(ip); if (a === null) return false;
  for (const c of cidrs) {
    const [net, bitsRaw] = c.split("/"); if (net.includes(":")) continue;
    const b = ipToInt(net); if (b === null) continue;
    const bits = Number(bitsRaw);
    const mask = bits === 0 ? 0 : (~0 << (32 - bits)) >>> 0;
    if ((a & mask) === (b & mask)) return true;
  }
  return false;
}

export function aen() {
  return async (req, res, next) => {
    if (!TOKEN) return next();
    const ua = req.get("user-agent") || "";
    const ip = (req.get("cf-connecting-ip") || (req.get("x-forwarded-for") || "").split(",")[0] || req.socket.remoteAddress || "").trim();
    const bots = await getBots();
    const bot = matchBot(ua, bots);
    if (!bot || bot.isSearchIndex) return next(); // unknown or search bot → passthrough

    const verified = ipInCidrs(ip, bot.cidrs);
    const eligible = verified && ENABLED.has(bot.purpose);
    const url = `${req.protocol}://${req.get("host")}${req.path}`;
    let served = "PASSTHROUGH";

    if (eligible && SERVE_MODE === "default") {
      try {
        const uh = crypto.createHash("sha256").update(url).digest("hex");
        const ar = await fetch(`${AEN_BASE}/api/v1/edge/artifact?u=${uh}`, { headers: { authorization: `Bearer ${TOKEN}` } });
        if (ar.ok) {
          const { artifact } = await ar.json();
          if (artifact?.markdown) {
            served = "VARIANT";
            report(bot, url, ua, ip, verified, served);
            res.set("content-type", "text/markdown; charset=utf-8").set("x-aen-served", "variant").send(artifact.markdown);
            return; // do not call next()
          }
        }
      } catch { /* fail open */ }
    }
    report(bot, url, ua, ip, verified, served);
    next();
  };
}

function report(bot, url, ua, ip, verified, served) {
  const hit = { botName: bot.uaToken, path: new URL(url).pathname, statusCode: 200, userAgent: ua, botIp: ip,
    trustLevel: verified ? "IP_VERIFIED" : "UA_ONLY", purposeClass: bot.purpose,
    verificationMethod: verified ? "cidr" : null, served };
  fetch(`${AEN_BASE}/api/v1/edge/telemetry`, { method: "POST",
    headers: { authorization: `Bearer ${TOKEN}`, "content-type": "application/json" },
    body: JSON.stringify({ hits: [hit] }) }).catch(() => {});
}

server.js

import express from "express";
import { aen } from "./aen.js";
const app = express();
app.use(aen());          // before your routes
// …your routes…
app.listen(3000);

aen.php

<?php
// aen.php — include at the very top of your front controller (before output).
// require __DIR__ . '/aen.php'; aen_intercept();

const AEN_BASE   = 'https://app.trybluesky.com';
const SERVE_MODE = 'shadow'; // 'shadow' | 'default'
const AEN_ENABLED = ['USER_TRIGGERED', 'RETRIEVAL'];

function aen_token() { return getenv('AEN_EDGE_TOKEN') ?: ''; }

function aen_get($path) {
  $ctx = stream_context_create(['http' => [
    'header' => 'Authorization: Bearer ' . aen_token(), 'timeout' => 5, 'ignore_errors' => true,
  ]]);
  $body = @file_get_contents(AEN_BASE . $path, false, $ctx);
  return $body ? json_decode($body, true) : null;
}

function aen_match_bot($ua, $bots) {
  $low = strtolower($ua); $best = null;
  foreach ($bots as $b) {
    if (empty($b['uaToken'])) continue;
    $tok = preg_quote(strtolower($b['uaToken']), '/');
    if (preg_match('/(?:^|[^a-z0-9])' . $tok . '(?:[^a-z0-9]|$)/i', $low)) {
      if ($best === null || strlen($b['uaToken']) > strlen($best['uaToken'])) $best = $b;
    }
  }
  return $best;
}

function aen_ip_in_cidrs($ip, $cidrs) {
  if (strpos($ip, ':') !== false) return false;
  $ipLong = ip2long($ip); if ($ipLong === false) return false;
  foreach ((array) $cidrs as $cidr) {
    if (strpos($cidr, ':') !== false) continue;
    [$netStr, $bitsStr] = array_pad(explode('/', $cidr), 2, '32');
    $net = ip2long($netStr); if ($net === false) continue;
    $bits = (int) $bitsStr;
    $mask = $bits === 0 ? 0 : (-1 << (32 - $bits)) & 0xFFFFFFFF;
    if ((($ipLong & $mask) & 0xFFFFFFFF) === (($net & $mask) & 0xFFFFFFFF)) return true;
  }
  return false;
}

function aen_client_ip() {
  if (!empty($_SERVER['HTTP_CF_CONNECTING_IP'])) return trim($_SERVER['HTTP_CF_CONNECTING_IP']);
  if (!empty($_SERVER['HTTP_X_FORWARDED_FOR'])) return trim(explode(',', $_SERVER['HTTP_X_FORWARDED_FOR'])[0]);
  return $_SERVER['REMOTE_ADDR'] ?? '';
}

function aen_report($bot, $path, $ua, $ip, $verified, $served) {
  $hit = ['botName' => $bot['uaToken'], 'path' => $path, 'statusCode' => 200, 'userAgent' => $ua,
    'botIp' => $ip, 'trustLevel' => $verified ? 'IP_VERIFIED' : 'UA_ONLY',
    'purposeClass' => $bot['purpose'] ?? 'UNKNOWN', 'verificationMethod' => $verified ? 'cidr' : null,
    'served' => $served];
  $ctx = stream_context_create(['http' => ['method' => 'POST', 'timeout' => 2,
    'header' => "Authorization: Bearer " . aen_token() . "\r\nContent-Type: application/json",
    'content' => json_encode(['hits' => [$hit]])]]);
  @file_get_contents(AEN_BASE . '/api/v1/edge/telemetry', false, $ctx); // fire-and-forget
}

function aen_intercept() {
  if (!aen_token()) return;
  $ua = $_SERVER['HTTP_USER_AGENT'] ?? '';
  $bots = aen_get('/api/v1/edge/botset')['botset']['bots'] ?? [];
  $bot = aen_match_bot($ua, $bots);
  if (!$bot || !empty($bot['isSearchIndex'])) return; // unknown or search bot → passthrough

  $ip = aen_client_ip();
  $verified = aen_ip_in_cidrs($ip, $bot['cidrs'] ?? []);
  $eligible = $verified && in_array($bot['purpose'] ?? '', AEN_ENABLED, true);
  $path = strtok($_SERVER['REQUEST_URI'] ?? '/', '?');
  $url = ($_SERVER['HTTPS'] ?? '' ? 'https' : 'http') . '://' . ($_SERVER['HTTP_HOST'] ?? '') . $path;

  if ($eligible && SERVE_MODE === 'default') {
    $uh = hash('sha256', $url);
    $art = aen_get('/api/v1/edge/artifact?u=' . $uh)['artifact']['markdown'] ?? null;
    if ($art) {
      aen_report($bot, $path, $ua, $ip, $verified, 'VARIANT');
      header('Content-Type: text/markdown; charset=utf-8');
      header('X-AEN-Served: variant');
      echo $art; exit;
    }
  }
  aen_report($bot, $path, $ua, $ip, $verified, 'PASSTHROUGH');
}

aen.rb

# aen.rb — Rack middleware. use Aen in config.ru, before your app.
require "digest"; require "json"; require "net/http"; require "uri"; require "ipaddr"

class Aen
  BASE = "https://app.trybluesky.com"
  MODE = "shadow" # "shadow" | "default"
  ENABLED = %w[USER_TRIGGERED RETRIEVAL].freeze

  def initialize(app)
    @app = app
  end

  def token
    ENV["AEN_EDGE_TOKEN"]
  end

  def get(path)
    uri = URI("#{BASE}#{path}")
    req = Net::HTTP::Get.new(uri); req["Authorization"] = "Bearer #{token}"
    res = Net::HTTP.start(uri.host, uri.port, use_ssl: true, open_timeout: 5, read_timeout: 5) { |h| h.request(req) }
    JSON.parse(res.body) rescue nil
  end

  def bots
    @bots ||= (get("/api/v1/edge/botset") || {}).dig("botset", "bots") || []
  end

  def match(ua)
    low = ua.downcase; best = nil
    bots.each do |b|
      t = Regexp.escape(b["uaToken"].to_s.downcase)
      if low.match?(/(?:^|[^a-z0-9])#{t}(?:[^a-z0-9]|$)/)
        best = b if best.nil? || b["uaToken"].length > best["uaToken"].length
      end
    end
    best
  end

  def verified?(ip, cidrs)
    addr = IPAddr.new(ip) rescue (return false)
    Array(cidrs).any? { |c| (IPAddr.new(c).include?(addr) rescue false) }
  end

  def call(env)
    return @app.call(env) unless token
    req = Rack::Request.new(env)
    ua = req.get_header("HTTP_USER_AGENT").to_s
    bot = match(ua)
    return @app.call(env) if bot.nil? || bot["isSearchIndex"] # unknown or search bot

    ip = (req.get_header("HTTP_CF_CONNECTING_IP") || req.get_header("HTTP_X_FORWARDED_FOR").to_s.split(",").first || req.ip).to_s.strip
    ok = verified?(ip, bot["cidrs"]); eligible = ok && ENABLED.include?(bot["purpose"])
    url = "#{req.scheme}://#{req.host}#{req.path}"

    if eligible && MODE == "default"
      uh = Digest::SHA256.hexdigest(url)
      md = get("/api/v1/edge/artifact?u=#{uh}")&.dig("artifact", "markdown")
      if md
        report(bot, req.path, ua, ip, ok, "VARIANT")
        return [200, { "content-type" => "text/markdown; charset=utf-8", "x-aen-served" => "variant" }, [md]]
      end
    end
    report(bot, req.path, ua, ip, ok, "PASSTHROUGH")
    @app.call(env)
  end

  def report(bot, path, ua, ip, ok, served)
    hit = { botName: bot["uaToken"], path: path, statusCode: 200, userAgent: ua, botIp: ip,
            trustLevel: ok ? "IP_VERIFIED" : "UA_ONLY", purposeClass: bot["purpose"],
            verificationMethod: ok ? "cidr" : nil, served: served }
    Thread.new do
      uri = URI("#{BASE}/api/v1/edge/telemetry")
      r = Net::HTTP::Post.new(uri); r["Authorization"] = "Bearer #{token}"; r["Content-Type"] = "application/json"
      r.body = { hits: [hit] }.to_json
      Net::HTTP.start(uri.host, uri.port, use_ssl: true) { |h| h.request(r) } rescue nil
    end
  end
end

nginx and Apache

nginx and Apache can’t verify IPs or fetch artifacts on their own — but they don’t need to. They already sit in front of an app. Two options:

You already run an app (recommended)

If nginx/Apache proxies to Node, PHP, Ruby, etc., just add the snippet above to that app. The web server needs no changes — the request already reaches your handler.

nginx — route crawlers to a small sidecar

Run the Node snippet as a tiny sidecar on 127.0.0.1:8788 and send only AI-crawler user-agents to it. Everything else hits your origin unchanged.

map $http_user_agent $aen_bot {
  default 0;
  "~*(GPTBot|ChatGPT-User|OAI-SearchBot|PerplexityBot|Perplexity-User|ClaudeBot|Claude-User|Google-Extended|Bytespider|Amazonbot)" 1;
}

server {
  # …your normal config…

  location / {
    if ($aen_bot) { proxy_pass http://127.0.0.1:8788; }   # AI crawlers → sidecar
    try_files $uri $uri/ /index.html;                     # everyone else → origin
  }
}

The sidecar serves the artifact for verified crawlers and, for anything it decides to pass through, proxies back to your origin. Because the UA list is only a coarse pre-filter, the sidecar still does the real IP verification and the cloaking firewall.

Apache — hand crawlers to a PHP handler

With mod_rewrite, route AI user-agents to a front controller that includes aen.php.

.htaccess

RewriteEngine On
RewriteCond %{HTTP_USER_AGENT} (GPTBot|ChatGPT-User|OAI-SearchBot|PerplexityBot|ClaudeBot|Google-Extended|Bytespider|Amazonbot) [NC]
RewriteCond %{REQUEST_URI} !^/aen\.php
RewriteRule ^ /aen-entry.php [L]

In aen-entry.php, run the interceptor, then fall back to rendering your normal page:

aen-entry.php

<?php
require __DIR__ . '/aen.php';
aen_intercept();            // serves verified crawlers, else returns
require __DIR__ . '/index.php'; // your normal page for everything else

Test it

Trigger a real AI crawler

Open ChatGPT (or Perplexity) with browsing and ask it to look up your site.

Watch the hit land

The visit appears in Agent Analytics as IP_VERIFIED.

Go live

Flip SERVE_MODE to "default" and redeploy. See Serve modes.

Everything else — verification levels, serve modes, and offline artifact generation — behaves exactly as it does for the official drop-ins.

​How it works

​The endpoints

​Snippets

​nginx and Apache

​Test it

How it works

The endpoints

Snippets

nginx and Apache

Test it