Sitemap for the public pages, linked from robots.txt #27
@@ -1,5 +1,7 @@
|
|||||||
import type { MetadataRoute } from "next";
|
import type { MetadataRoute } from "next";
|
||||||
|
|
||||||
|
import { siteUrl } from "@/lib/share";
|
||||||
|
|
||||||
// AI crawlers that train models or answer from the site: they load pages
|
// AI crawlers that train models or answer from the site: they load pages
|
||||||
// far more often than visitors do. Link preview robots (Facebook, Messenger,
|
// far more often than visitors do. Link preview robots (Facebook, Messenger,
|
||||||
// Slack, WhatsApp, X, LinkedIn) stay allowed, they make the shared links work.
|
// Slack, WhatsApp, X, LinkedIn) stay allowed, they make the shared links work.
|
||||||
@@ -38,8 +40,9 @@ const AI_CRAWLERS = [
|
|||||||
"Scrapy",
|
"Scrapy",
|
||||||
];
|
];
|
||||||
|
|
||||||
export default function robots(): MetadataRoute.Robots {
|
export default async function robots(): Promise<MetadataRoute.Robots> {
|
||||||
return {
|
return {
|
||||||
|
sitemap: `${await siteUrl()}/sitemap.xml`,
|
||||||
rules: [
|
rules: [
|
||||||
{ userAgent: AI_CRAWLERS, disallow: "/" },
|
{ userAgent: AI_CRAWLERS, disallow: "/" },
|
||||||
// Logged-in pages redirect anyway; shared links carry noindex.
|
// Logged-in pages redirect anyway; shared links carry noindex.
|
||||||
|
|||||||
@@ -0,0 +1,15 @@
|
|||||||
|
import type { MetadataRoute } from "next";
|
||||||
|
|
||||||
|
import { siteUrl } from "@/lib/share";
|
||||||
|
|
||||||
|
/** The public pages for search engines; everything else needs a login. */
|
||||||
|
export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
|
||||||
|
const base = await siteUrl();
|
||||||
|
const privacy = { languages: { hu: `${base}/adatvedelem`, en: `${base}/privacy` } };
|
||||||
|
return [
|
||||||
|
{ url: `${base}/`, changeFrequency: "weekly", priority: 1 },
|
||||||
|
{ url: `${base}/pelda`, changeFrequency: "monthly", priority: 0.8 },
|
||||||
|
{ url: `${base}/adatvedelem`, changeFrequency: "yearly", priority: 0.2, alternates: privacy },
|
||||||
|
{ url: `${base}/privacy`, changeFrequency: "yearly", priority: 0.2, alternates: privacy },
|
||||||
|
];
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user