feat: update game cache per file instead of full re-download

apps/client/README.md

@@ -52,7 +52,7 @@ The client loader exposes the following HTTP routes:
 | --------------- | --------------------------------------------------------------------- |
 | `GET /`         | Serves the `loader-website` HTML application                          |
 | `GET /*`        | Serves static assets bundled with `loader-website`                    |
-| `GET /manifest` | Returns the list of game files and the current version as JSON        |
+| `GET /manifest` | Returns the game files (path, SHA-256 hash, size) and version as JSON |
 | `GET /env`      | Returns `NANOFORGE_*` environment variables (prefix stripped) as JSON |
 | `GET /game/*`   | Serves compiled game client files from the configured game directory  |
 

apps/client/src/files.ts

@@ -1,23 +1,40 @@
 import { readdir } from "node:fs/promises";
 
-import { MANIFEST } from "./server";
+import { type IManifestFile } from "./server";
 
-export const updateFiles = (dir: string) => {
-  return addPath(dir, "");
+const HASHES = new Map<string, { stamp: string; hash: string }>();
+
+export const updateFiles = async (dir: string) => {
+  const files: IManifestFile[] = [];
+  await addPath(files, dir, "");
+  return files;
 };
 
-const addPath = async (path: string, exportedPath: string) => {
+const addPath = async (files: IManifestFile[], path: string, exportedPath: string) => {
   try {
-    if ((await Bun.file(path).stat()).isDirectory()) {
+    const stat = await Bun.file(path).stat();
+    if (stat.isDirectory()) {
       for (const file of await readdir(path)) {
-        await addPath(`${path}/${file}`, `${exportedPath}/${file}`);
+        await addPath(files, `${path}/${file}`, `${exportedPath}/${file}`);
       }
       return;
     }
-    MANIFEST.files.push({
+    files.push({
       path: exportedPath,
+      hash: await getHash(path, `${stat.size}:${stat.mtimeMs}`),
+      size: stat.size,
     });
   } catch (e) {
     console.error(e);
   }
 };
+
+const getHash = async (path: string, stamp: string) => {
+  const cached = HASHES.get(path);
+  if (cached?.stamp === stamp) return cached.hash;
+  const hash = new Bun.CryptoHasher("sha256")
+    .update(await Bun.file(path).arrayBuffer())
+    .digest("hex");
+  HASHES.set(path, { stamp, hash });
+  return hash;
+};

apps/client/src/manifest.ts

@@ -1,31 +1,20 @@
 import { updateFiles } from "./files";
-import { MANIFEST } from "./server";
+import { type IManifest, type IManifestFile, MANIFEST } from "./server";
 
-export const updateManifest = async (dir: string) => {
-  MANIFEST.files = [];
-  await updateFiles(dir);
-  MANIFEST.version = await getVersion(dir);
+export const updateManifest = async (dir: string): Promise<IManifest> => {
+  const files = await updateFiles(dir);
+  return { ...MANIFEST, files, version: await getVersion(files) };
 };
 
-const getVersion = async (dir: string) => {
+const getVersion = async (files: IManifestFile[]) => {
   try {
     return await Bun.file("public/version").text();
   } catch {
-    return await fingerprintFiles(dir);
+    return fingerprintFiles(files);
   }
 };
 
-const fingerprintFiles = async (dir: string): Promise<string> => {
-  const stamps = await Promise.all(
-    MANIFEST.files.map(async ({ path }) => {
-      try {
-        const stat = await Bun.file(`${dir}${path}`).stat();
-        return `${path}:${stat.size}:${stat.mtimeMs}`;
-      } catch {
-        return `${path}:missing`;
-      }
-    }),
-  );
-  stamps.sort();
+const fingerprintFiles = (files: IManifestFile[]): string => {
+  const stamps = files.map(({ path, hash }) => `${path}:${hash}`).sort();
   return Bun.hash(stamps.join("|")).toString(16);
 };

apps/client/src/server.ts

@@ -5,9 +5,11 @@ import { getGameEnv } from "./env";
 import { updateManifest } from "./manifest";
 import { startWatch } from "./watch";
 
-type IManifest = {
+export type IManifestFile = { path: string; hash: string; size: number };
+
+export type IManifest = {
   version: string;
-  files: { path: string }[];
+  files: IManifestFile[];
   watch: { enable: false } | { enable: true; url: string };
 };
 
@@ -35,9 +37,7 @@ const { port, dir, watch, watchPort, watchServerDir, cert, key } = program.opts<
 
 if (!dir) throw new Error("No game dir specified");
 
-export const MANIFEST: IManifest = {
-  version: "",
-  files: [],
+export const MANIFEST: Pick<IManifest, "watch"> = {
   watch: {
     enable: false,
   },
@@ -86,8 +86,7 @@ const server = Bun.serve({
     },
     "/manifest": async () => {
       try {
-        await updateManifest(dir);
-        return Response.json(MANIFEST, {
+        return Response.json(await updateManifest(dir), {
           headers,
         });
       } catch (error) {

apps/website/README.md

@@ -49,8 +49,8 @@ nf start
 When the browser opens the loader URL, the following steps happen in order:
 
 1. **Fetch manifest** — requests `/manifest` to get the game version and the list of files.
-2. **Verify cache** — checks whether the game files from the previous session are still present in the browser's Origin Private File System (OPFS).
-3. **Download files** — if the cache is stale or missing, downloads each game file from `/game/*` and writes it into OPFS, showing a progress bar and the current file name.
+2. **Verify cache** — compares each file's hash and size with the copy cached in the browser's Origin Private File System (OPFS), and removes cached files no longer in the manifest.
+3. **Download files** — downloads only new or changed files from `/game/*` and writes them into OPFS, showing a progress bar and the current file name.
 4. **Fetch environment** — requests `/env` to get the `NANOFORGE_*` variables forwarded by the server.
 5. **Bootstrap game** — dynamically imports `/main.js` from the local cache, calls its exported `main({ files, env, container })` and hides the loading screen.
 

apps/website/src/cache/cache.ts

@@ -1,69 +1,91 @@
 import { FileSystemManager } from "../file-system";
-import { isManifestUpToDate } from "../manifest";
 import {
   type IExtendedManifest,
   type IExtendedManifestFile,
   type IManifest,
   type IManifestFile,
 } from "../types/manifest.type";
 import { Logger } from "../utils/logger.utils";
-import { setVersion } from "../version";
 import { setLoadingStatus, setLoadingTotalFiles } from "../window";
 
 const CACHE_LOCK_NAME = "nanoforge-game-cache";
+const CACHE_HASHES_KEY = "game-files";
+
+type IFileHashes = Record<string, string>;
 
 async function withCacheLock<T>(fn: () => Promise<T>): Promise<T> {
   if (typeof navigator === "undefined" || !navigator.locks) return fn();
   return await navigator.locks.request(CACHE_LOCK_NAME, fn);
 }
 
+const getFileHashes = (): IFileHashes => {
+  try {
+    return JSON.parse(localStorage.getItem(CACHE_HASHES_KEY) ?? "{}");
+  } catch {
+    return {};
+  }
+};
+
+const setFileHashes = (hashes: IFileHashes) => {
+  localStorage.setItem(CACHE_HASHES_KEY, JSON.stringify(hashes));
+};
+
 export class GameCache {
   private readonly logger: Logger = new Logger("Cache");
   private readonly fs: FileSystemManager = new FileSystemManager("game");
 
   async updateCache(manifest: IManifest, force = false): Promise<IExtendedManifest> {
     this.logger.info("Starting cache game files");
-
-    let extendedManifest = await this._tryReuseCache(manifest, force);
-
-    if (!extendedManifest) {
-      extendedManifest = await withCacheLock(async () => {
-        return (
-          (await this._tryReuseCache(manifest, force)) ?? (await this._updateCacheProcess(manifest))
-        );
-      });
-    }
-
-    setVersion(manifest.version);
+    const extendedManifest = await withCacheLock(() =>
+      this._updateCacheProcess(manifest.files, force),
+    );
     this.logger.info("Game files cached");
     return extendedManifest;
   }
 
-  private async _tryReuseCache(
-    manifest: IManifest,
+  private async _updateCacheProcess(
+    files: IManifestFile[],
     force: boolean,
-  ): Promise<IExtendedManifest | undefined> {
-    if (force || !isManifestUpToDate(manifest)) return undefined;
-    return this._parseCache(manifest);
-  }
+  ): Promise<IExtendedManifest> {
+    const stored = force ? {} : getFileHashes();
 
-  private async _updateCacheProcess(manifest: IManifest): Promise<IExtendedManifest> {
-    setLoadingStatus("Cleaning cache game files");
-    const directory = await this.fs.getDirectory();
-    await directory.clear();
-    return {
-      files: await this._updateCacheFiles(manifest.files),
-    };
-  }
+    if (!Object.keys(stored).length) {
+      setLoadingStatus("Cleaning cache game files");
+      setFileHashes(stored);
+      await (await this.fs.getDirectory()).clear();
+    }
 
-  private async _updateCacheFiles(files: IManifestFile[]): Promise<IExtendedManifestFile[]> {
-    const res = [];
-    setLoadingTotalFiles(files.length);
+    const remotePaths = new Set(files.map(({ path }) => path));
+    for (const path of Object.keys(stored)) {
+      if (!remotePaths.has(path)) await this._removeCacheFile(path);
+    }
+    const hashes: IFileHashes = Object.fromEntries(
+      Object.entries(stored).filter(([path]) => remotePaths.has(path)),
+    );
+    setFileHashes(hashes);
+
+    setLoadingStatus("Verifying application integrity");
+    const res: IExtendedManifestFile[] = [];
+    const outdated: number[] = [];
     for (const [i, file] of files.entries()) {
-      setLoadingStatus(`Download: ${file.path.replace(/^\/+/, "")}`, i);
-      res.push(await this._updateCacheFile(file));
+      const cached = await this._parseFile(file, hashes);
+      if (cached) res[i] = cached;
+      else outdated.push(i);
     }
-    return res;
+
+    setLoadingTotalFiles(outdated.length);
+    for (const [n, i] of outdated.entries()) {
+      const file = files[i] as IManifestFile;
+      setLoadingStatus(
+        `Download ${n + 1} / ${outdated.length}: ${file.path.replace(/^\/+/, "")}`,
+        n,
+      );
+      res[i] = await this._updateCacheFile(file);
+      hashes[file.path] = file.hash;
+      setFileHashes(hashes);
+    }
+
+    return { files: res };
   }
 
   private async _updateCacheFile(fileManifest: IManifestFile): Promise<IExtendedManifestFile> {
@@ -97,35 +119,25 @@ export class GameCache {
     };
   }
 
-  private async _parseCache(manifest: IManifest): Promise<IExtendedManifest | undefined> {
-    setLoadingStatus("Verifying application integrity");
-    return this._parseFiles(manifest.files);
-  }
-
-  private async _parseFiles(files: IManifestFile[]): Promise<IExtendedManifest | undefined> {
-    const res = [];
-    for (const file of files) {
-      const r = await this._parseFile(file);
-      if (!r) return undefined;
-      res.push(r);
-    }
-    return {
-      files: res,
-    };
+  private async _removeCacheFile(path: string): Promise<void> {
+    const i = path.lastIndexOf("/");
+    await this.fs
+      .getDirectory(path.slice(0, i))
+      .then((directory) => directory.removeChild(path.slice(i + 1)))
+      .catch(() => undefined);
   }
 
-  /**
-   * @todo Function to verify if the file passed in parameters exist
-   * @todo Verify if the file passed in parameters is really complete
-   */
   private async _parseFile(
     fileManifest: IManifestFile,
+    hashes: IFileHashes,
   ): Promise<IExtendedManifestFile | undefined> {
+    if (hashes[fileManifest.path] !== fileManifest.hash) return undefined;
     try {
-      const file = await this.fs.getFile(fileManifest.path);
+      const file = await (await this.fs.getFile(fileManifest.path)).getFile();
+      if (file.size !== fileManifest.size) return undefined;
       return {
         ...fileManifest,
-        localPath: await file.getUrl(),
+        localPath: URL.createObjectURL(file),
       };
     } catch {
       return undefined;

apps/website/src/manifest.ts

@@ -1,5 +1,4 @@
 import { type IManifest } from "./types/manifest.type";
-import { getVersion } from "./version";
 import { setLoadingStatus } from "./window";
 
 export const getManifest = async (): Promise<IManifest> => {
@@ -10,10 +9,3 @@ export const getManifest = async (): Promise<IManifest> => {
   }
   return await res.json();
 };
-
-export const isManifestUpToDate = (manifest: IManifest): boolean => {
-  setLoadingStatus("Verifying manifest");
-  const currentVersion = getVersion();
-  if (!currentVersion || currentVersion !== manifest.version) return false;
-  return true;
-};

apps/website/src/types/manifest.type.ts

@@ -1,5 +1,7 @@
 export interface IManifestFile {
   path: string;
+  hash: string;
+  size: number;
 }
 
 export interface IManifest {

docs/docs/loader/2-client.mdx

@@ -27,7 +27,7 @@ npx loader-client --dir .nanoforge/client --port 3000
 | --------------- | --------------------------------------------------------------------- |
 | `GET /`         | Serves the `loader-website` HTML application                          |
 | `GET /*`        | Serves static assets bundled with `loader-website`                    |
-| `GET /manifest` | Returns the list of game files and the current version as JSON        |
+| `GET /manifest` | Returns the game files (path, SHA-256 hash, size) and version as JSON |
 | `GET /env`      | Returns `NANOFORGE_*` environment variables (prefix stripped) as JSON |
 | `GET /game/*`   | Serves compiled game client files from the configured game directory  |
 

docs/docs/loader/3-website.mdx

@@ -16,8 +16,8 @@ This package is a dependency of `loader-client` and is not consumed directly.
 When the browser opens the loader URL, the following steps happen in order:
 
 1. **Fetch manifest** — requests `/manifest` to get the game version and the list of files to download.
-2. **Verify cache** — checks whether the game files from the previous session are still present in the browser's Origin Private File System (OPFS). The loading screen shows _"Verifying application integrity"_.
-3. **Download files** — if the cache is stale or missing, downloads each game file from `/game/*` and writes it into OPFS. The loading screen shows a progress bar and the name of the current file.
+2. **Verify cache** — compares each file's hash and size with the copy cached in the browser's Origin Private File System (OPFS), and removes cached files no longer in the manifest. The loading screen shows _"Verifying application integrity"_.
+3. **Download files** — downloads only new or changed files from `/game/*` and writes them into OPFS. The loading screen shows a progress bar and the name of the current file.
 4. **Fetch environment** — requests `/env` to get the `NANOFORGE_*` variables forwarded by `loader-client`.
 5. **Bootstrap game** — dynamically imports `/main.js` from the local OPFS cache, calls its exported `main({ files, env, container })`, and fades out the loading screen.