Sitemap for the public pages, linked from robots.txt
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
import type { MetadataRoute } from "next";
|
||||
|
||||
import { siteUrl } from "@/lib/share";
|
||||
|
||||
// AI crawlers that train models or answer from the site: they load pages
|
||||
// far more often than visitors do. Link preview robots (Facebook, Messenger,
|
||||
// Slack, WhatsApp, X, LinkedIn) stay allowed, they make the shared links work.
|
||||
@@ -38,8 +40,9 @@ const AI_CRAWLERS = [
|
||||
"Scrapy",
|
||||
];
|
||||
|
||||
export default function robots(): MetadataRoute.Robots {
|
||||
export default async function robots(): Promise<MetadataRoute.Robots> {
|
||||
return {
|
||||
sitemap: `${await siteUrl()}/sitemap.xml`,
|
||||
rules: [
|
||||
{ userAgent: AI_CRAWLERS, disallow: "/" },
|
||||
// Logged-in pages redirect anyway; shared links carry noindex.
|
||||
|
||||
Reference in New Issue
Block a user