feat: robots.txt

src/app/robots.ts

@@ -5,7 +5,7 @@ export default function robots(): MetadataRoute.Robots {
     rules: {
       userAgent: "*",
       allow: "/",
-      disallow: ["/dashboard/", "/preview/", "/api/"],
+      disallow: ["/dashboard/", "/preview/"],
     },
   }
 }

src/app/site/[site]/robots.txt/route.ts

@@ -0,0 +1,19 @@
+import { NextRequest } from "next/server"
+
+import { NextServerResponse } from "~/lib/server-helper"
+
+export const GET = async (req: NextRequest) => {
+  const response = new NextServerResponse()
+
+  return response.headers({
+    "Content-Type": "text/plain",
+    "Access-Control-Allow-Methods": "GET",
+    "Access-Control-Allow-Origin": "*",
+  }).text(`User-Agent: *
+Allow: /
+Disallow: /dashboard/
+Disallow: /preview/
+Disallow: /api/
+
+Sitemap: https://${req.headers.get("host")}/sitemap.xml`)
+}

src/app/site/[site]/sitemap.xml/route.ts

@@ -36,6 +36,7 @@ export const GET = async (req: NextRequest) => {
       characterId: site?.characterId,
       type: "post",
       visibility: PageVisibilityEnum.Published,
+      limit: 1000,
     },
     queryClient,
   )

src/middleware.ts

@@ -55,8 +55,7 @@ export default async function middleware(req: NextRequest) {
     pathname.startsWith("/locales/") ||
     pathname.match(/^\/(workbox|worker|fallback)-\w+\.js(\.map)?$/) ||
     pathname === "/sw.js" ||
-    pathname === "/sw.js.map" ||
-    pathname === "/robots.txt"
+    pathname === "/sw.js.map"
   ) {
     return NextResponse.next()
   }