import type { MetadataRoute } from "next"; // AI crawlers that train models or answer from the site: they load pages // far more often than visitors do. Link preview robots (Facebook, Messenger, // Slack, WhatsApp, X, LinkedIn) stay allowed, they make the shared links work. const AI_CRAWLERS = [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "anthropic-ai", "CCBot", "Google-Extended", "GoogleOther", "Applebot-Extended", "PerplexityBot", "Perplexity-User", "Bytespider", "Amazonbot", "meta-externalagent", "meta-externalfetcher", "FacebookBot", "cohere-ai", "cohere-training-data-crawler", "Diffbot", "ImagesiftBot", "Omgilibot", "Omgili", "YouBot", "Timpibot", "AI2Bot", "Ai2Bot-Dolma", "DuckAssistBot", "MistralAI-User", "PetalBot", "Scrapy", ]; export default function robots(): MetadataRoute.Robots { return { rules: [ { userAgent: AI_CRAWLERS, disallow: "/" }, // Logged-in pages redirect anyway; shared links carry noindex. { userAgent: "*", allow: "/", disallow: ["/api/", "/admin", "/profil", "/kepletek", "/osszevetesek", "/szemelyek"] }, ], }; }