robots.txt: block only AI training crawlers, let AI search in slowly
CI / test (push) Successful in 1m44s
CI / build (push) Successful in 5m55s
CI / deploy (push) Skipped

AI search engines and chat fetchers (OAI-SearchBot, ChatGPT-User,
Claude-SearchBot, PerplexityBot, ...) may now read the public pages,
with Crawl-delay: 10, so they can cite the site. Training crawlers stay
disallowed. Docs: the matching Cloudflare AI policy settings, and the
referral note no longer mentions localStorage.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
feco
2026-11-22 05:56:41 +00:00
co-authored by Claude Opus 5.5
parent 116e4747c2
commit a6ddfc6924
3 changed files with 42 additions and 20 deletions
+34 -17
View File
@@ -2,27 +2,21 @@ import type { MetadataRoute } from "next";
import { siteUrl } from "@/lib/share";
// AI crawlers that train models or answer from the site: they load pages
// far more often than visitors do. Link preview robots (Facebook, Messenger,
// Slack, WhatsApp, X, LinkedIn) stay allowed, they make the shared links work.
const AI_CRAWLERS = [
// Crawlers that collect content to train AI models, or to sell it as
// training data. They bring no visitors back, so they are shut out.
// Google-Extended and Applebot-Extended only opt out of training: Google
// Search (AI Overviews included) and Siri keep reading the site.
const TRAINING_CRAWLERS = [
"GPTBot",
"ChatGPT-User",
"OAI-SearchBot",
"ClaudeBot",
"Claude-User",
"Claude-SearchBot",
"anthropic-ai",
"CCBot",
"Google-Extended",
"GoogleOther",
"Applebot-Extended",
"PerplexityBot",
"Perplexity-User",
"Bytespider",
"Amazonbot",
"meta-externalagent",
"meta-externalfetcher",
"FacebookBot",
"cohere-ai",
"cohere-training-data-crawler",
@@ -30,23 +24,46 @@ const AI_CRAWLERS = [
"ImagesiftBot",
"Omgilibot",
"Omgili",
"YouBot",
"Webzio-Extended",
"Timpibot",
"AI2Bot",
"Ai2Bot-Dolma",
"DuckAssistBot",
"MistralAI-User",
"PanguBot",
"Kangaroo Bot",
"img2dataset",
// Not training, but heavy and of no use here: Huawei's Petal search and
// Scrapy's default user agent.
"PetalBot",
"Scrapy",
];
// AI search engines and the fetchers that read a page when someone asks a
// chatbot about it. These can cite the site and send visitors, so they may
// come, but at most one page every few seconds (for those that honour it).
const AI_SEARCH = [
"OAI-SearchBot",
"ChatGPT-User",
"Claude-SearchBot",
"Claude-User",
"PerplexityBot",
"Perplexity-User",
"DuckAssistBot",
"MistralAI-User",
"meta-externalfetcher",
"YouBot",
];
// Logged-in pages redirect anyway; shared links carry noindex.
const PRIVATE = ["/api/", "/admin", "/profil", "/kepletek", "/osszevetesek", "/szemelyek", "/szamlak"];
export default async function robots(): Promise<MetadataRoute.Robots> {
return {
sitemap: `${await siteUrl()}/sitemap.xml`,
rules: [
{ userAgent: AI_CRAWLERS, disallow: "/" },
// Logged-in pages redirect anyway; shared links carry noindex.
{ userAgent: "*", allow: "/", disallow: ["/api/", "/admin", "/profil", "/kepletek", "/osszevetesek", "/szemelyek", "/szamlak"] },
{ userAgent: TRAINING_CRAWLERS, disallow: "/" },
// A crawler reads only its own group, so the private paths are repeated.
{ userAgent: AI_SEARCH, allow: "/", disallow: PRIVATE, crawlDelay: 10 },
{ userAgent: "*", allow: "/", disallow: PRIVATE },
],
};
}