Only allow crawling a curated set of diff pages

src/app/robots.ts

@@ -1,6 +1,7 @@
 import type { MetadataRoute } from "next";
+import getCrawlableDiffs from "^/lib/crawlableDiffs";
 
-export default function robots(): MetadataRoute.Robots {
+export default async function robots(): Promise<MetadataRoute.Robots> {
     return {
         rules: [
             {
@@ -9,8 +10,8 @@ export default function robots(): MetadataRoute.Robots {
             },
             {
                 userAgent: "*",
-                allow: "/",
-                disallow: "/api/",
+                allow: ["/", ...(await getCrawlableDiffs())],
+                disallow: ["/api/", "/*..."],
                 crawlDelay: 10,
             },
         ],

src/lib/crawlableDiffs.test.ts

@@ -0,0 +1,24 @@
+import { selectCrawlableDiffs } from "./crawlableDiffs";
+
+describe("selectCrawlableDiffs", () => {
+    it("picks the latest major and minor bump", () => {
+        expect(
+            selectCrawlableDiffs("pkg", [
+                "1.0.0",
+                "1.1.0",
+                "1.1.1",
+                "2.0.0",
+                "2.1.0",
+                "2.1.1",
+            ]),
+        ).toEqual(["/pkg@1.1.1...pkg@2.0.0", "/pkg@2.0.0...pkg@2.1.0"]);
+    });
+
+    it("returns nothing for a single version", () => {
+        expect(selectCrawlableDiffs("pkg", ["1.0.0"])).toEqual([]);
+    });
+
+    it("skips bump types that don't exist", () => {
+        expect(selectCrawlableDiffs("pkg", ["1.0.0", "1.0.1"])).toEqual([]);
+    });
+});

src/lib/crawlableDiffs.ts

@@ -0,0 +1,68 @@
+import { cacheLife } from "next/cache";
+import getPackageVersions from "^/lib/api/packageVersions";
+import {
+    type Comparison,
+    generateComparisons,
+} from "^/lib/utils/generateComparisons";
+import specsToDiff from "^/lib/utils/specsToDiff";
+
+export const CRAWLABLE_PACKAGES = [
+    "react",
+    "express",
+    "lodash",
+    "axios",
+    "chalk",
+    "commander",
+    "semver",
+    "debug",
+    "uuid",
+    "zod",
+    "dotenv",
+    "yargs",
+] as const;
+
+const CRAWLABLE_BUMP_TYPES: Comparison["type"][] = ["major", "minor"];
+
+export function selectCrawlableDiffs(
+    packageName: string,
+    versions: string[],
+): string[] {
+    const comparisons = generateComparisons(versions);
+
+    return CRAWLABLE_BUMP_TYPES.flatMap((type) => {
+        const comparison = comparisons.find((c) => c.type === type);
+
+        return comparison
+            ? [
+                  `/${specsToDiff([
+                      `${packageName}@${comparison.from}`,
+                      `${packageName}@${comparison.to}`,
+                  ])}`,
+              ]
+            : [];
+    });
+}
+
+export default async function getCrawlableDiffs(): Promise<string[]> {
+    "use cache";
+
+    cacheLife("days");
+
+    const diffs = await Promise.all(
+        CRAWLABLE_PACKAGES.map(async (packageName) => {
+            try {
+                const versions = await getPackageVersions(packageName);
+
+                return versions
+                    ? selectCrawlableDiffs(packageName, Object.keys(versions))
+                    : [];
+            } catch (e) {
+                console.error(`[${packageName}] crawlable diffs error:`, e);
+
+                return [];
+            }
+        }),
+    );
+
+    return diffs.flat();
+}