From f7b37e30ca548fb0aa7b097533e6eec142ea4215 Mon Sep 17 00:00:00 2001 From: Leo Date: Fri, 23 Oct 2026 15:56:06 +0200 Subject: [PATCH] feat: update game cache per file instead of full re-download client: manifest files carry sha256 hash + size, hashes memoized by size+mtime, manifest built per request (no shared list race). website: diff local hash index vs manifest, download new/changed, delete removed, reuse only when hash and size match. Closes #8 --- apps/client/README.md | 2 +- apps/client/src/files.ts | 31 ++++-- apps/client/src/manifest.ts | 27 ++---- apps/client/src/server.ts | 13 ++- apps/website/README.md | 4 +- apps/website/src/cache/cache.ts | 122 +++++++++++++----------- apps/website/src/manifest.ts | 8 -- apps/website/src/types/manifest.type.ts | 2 + apps/website/src/version/index.ts | 1 - apps/website/src/version/version.ts | 13 --- docs/docs/loader/2-client.mdx | 2 +- docs/docs/loader/3-website.mdx | 4 +- 12 files changed, 113 insertions(+), 116 deletions(-) delete mode 100644 apps/website/src/version/index.ts delete mode 100644 apps/website/src/version/version.ts diff --git a/apps/client/README.md b/apps/client/README.md index cb95a29..7bd901f 100644 --- a/apps/client/README.md +++ b/apps/client/README.md @@ -52,7 +52,7 @@ The client loader exposes the following HTTP routes: | --------------- | --------------------------------------------------------------------- | | `GET /` | Serves the `loader-website` HTML application | | `GET /*` | Serves static assets bundled with `loader-website` | -| `GET /manifest` | Returns the list of game files and the current version as JSON | +| `GET /manifest` | Returns the game files (path, SHA-256 hash, size) and version as JSON | | `GET /env` | Returns `NANOFORGE_*` environment variables (prefix stripped) as JSON | | `GET /game/*` | Serves compiled game client files from the configured game directory | diff --git a/apps/client/src/files.ts b/apps/client/src/files.ts index ca63887..ef7e296 100644 --- a/apps/client/src/files.ts +++ b/apps/client/src/files.ts @@ -1,23 +1,40 @@ import { readdir } from "node:fs/promises"; -import { MANIFEST } from "./server"; +import { type IManifestFile } from "./server"; -export const updateFiles = (dir: string) => { - return addPath(dir, ""); +const HASHES = new Map(); + +export const updateFiles = async (dir: string) => { + const files: IManifestFile[] = []; + await addPath(files, dir, ""); + return files; }; -const addPath = async (path: string, exportedPath: string) => { +const addPath = async (files: IManifestFile[], path: string, exportedPath: string) => { try { - if ((await Bun.file(path).stat()).isDirectory()) { + const stat = await Bun.file(path).stat(); + if (stat.isDirectory()) { for (const file of await readdir(path)) { - await addPath(`${path}/${file}`, `${exportedPath}/${file}`); + await addPath(files, `${path}/${file}`, `${exportedPath}/${file}`); } return; } - MANIFEST.files.push({ + files.push({ path: exportedPath, + hash: await getHash(path, `${stat.size}:${stat.mtimeMs}`), + size: stat.size, }); } catch (e) { console.error(e); } }; + +const getHash = async (path: string, stamp: string) => { + const cached = HASHES.get(path); + if (cached?.stamp === stamp) return cached.hash; + const hash = new Bun.CryptoHasher("sha256") + .update(await Bun.file(path).arrayBuffer()) + .digest("hex"); + HASHES.set(path, { stamp, hash }); + return hash; +}; diff --git a/apps/client/src/manifest.ts b/apps/client/src/manifest.ts index 0970435..9ff4db5 100644 --- a/apps/client/src/manifest.ts +++ b/apps/client/src/manifest.ts @@ -1,31 +1,20 @@ import { updateFiles } from "./files"; -import { MANIFEST } from "./server"; +import { type IManifest, type IManifestFile, MANIFEST } from "./server"; -export const updateManifest = async (dir: string) => { - MANIFEST.files = []; - await updateFiles(dir); - MANIFEST.version = await getVersion(dir); +export const updateManifest = async (dir: string): Promise => { + const files = await updateFiles(dir); + return { ...MANIFEST, files, version: await getVersion(files) }; }; -const getVersion = async (dir: string) => { +const getVersion = async (files: IManifestFile[]) => { try { return await Bun.file("public/version").text(); } catch { - return await fingerprintFiles(dir); + return fingerprintFiles(files); } }; -const fingerprintFiles = async (dir: string): Promise => { - const stamps = await Promise.all( - MANIFEST.files.map(async ({ path }) => { - try { - const stat = await Bun.file(`${dir}${path}`).stat(); - return `${path}:${stat.size}:${stat.mtimeMs}`; - } catch { - return `${path}:missing`; - } - }), - ); - stamps.sort(); +const fingerprintFiles = (files: IManifestFile[]): string => { + const stamps = files.map(({ path, hash }) => `${path}:${hash}`).sort(); return Bun.hash(stamps.join("|")).toString(16); }; diff --git a/apps/client/src/server.ts b/apps/client/src/server.ts index 2d2d618..d25739b 100644 --- a/apps/client/src/server.ts +++ b/apps/client/src/server.ts @@ -5,9 +5,11 @@ import { getGameEnv } from "./env"; import { updateManifest } from "./manifest"; import { startWatch } from "./watch"; -type IManifest = { +export type IManifestFile = { path: string; hash: string; size: number }; + +export type IManifest = { version: string; - files: { path: string }[]; + files: IManifestFile[]; watch: { enable: false } | { enable: true; url: string }; }; @@ -35,9 +37,7 @@ const { port, dir, watch, watchPort, watchServerDir, cert, key } = program.opts< if (!dir) throw new Error("No game dir specified"); -export const MANIFEST: IManifest = { - version: "", - files: [], +export const MANIFEST: Pick = { watch: { enable: false, }, @@ -86,8 +86,7 @@ const server = Bun.serve({ }, "/manifest": async () => { try { - await updateManifest(dir); - return Response.json(MANIFEST, { + return Response.json(await updateManifest(dir), { headers, }); } catch (error) { diff --git a/apps/website/README.md b/apps/website/README.md index 098ae31..b66f606 100644 --- a/apps/website/README.md +++ b/apps/website/README.md @@ -49,8 +49,8 @@ nf start When the browser opens the loader URL, the following steps happen in order: 1. **Fetch manifest** — requests `/manifest` to get the game version and the list of files. -2. **Verify cache** — checks whether the game files from the previous session are still present in the browser's Origin Private File System (OPFS). -3. **Download files** — if the cache is stale or missing, downloads each game file from `/game/*` and writes it into OPFS, showing a progress bar and the current file name. +2. **Verify cache** — compares each file's hash and size with the copy cached in the browser's Origin Private File System (OPFS), and removes cached files no longer in the manifest. +3. **Download files** — downloads only new or changed files from `/game/*` and writes them into OPFS, showing a progress bar and the current file name. 4. **Fetch environment** — requests `/env` to get the `NANOFORGE_*` variables forwarded by the server. 5. **Bootstrap game** — dynamically imports `/main.js` from the local cache, calls its exported `main({ files, env, container })` and hides the loading screen. diff --git a/apps/website/src/cache/cache.ts b/apps/website/src/cache/cache.ts index 969dbeb..798ffbc 100644 --- a/apps/website/src/cache/cache.ts +++ b/apps/website/src/cache/cache.ts @@ -1,5 +1,4 @@ import { FileSystemManager } from "../file-system"; -import { isManifestUpToDate } from "../manifest"; import { type IExtendedManifest, type IExtendedManifestFile, @@ -7,63 +6,86 @@ import { type IManifestFile, } from "../types/manifest.type"; import { Logger } from "../utils/logger.utils"; -import { setVersion } from "../version"; import { setLoadingStatus, setLoadingTotalFiles } from "../window"; const CACHE_LOCK_NAME = "nanoforge-game-cache"; +const CACHE_HASHES_KEY = "game-files"; + +type IFileHashes = Record; async function withCacheLock(fn: () => Promise): Promise { if (typeof navigator === "undefined" || !navigator.locks) return fn(); return await navigator.locks.request(CACHE_LOCK_NAME, fn); } +const getFileHashes = (): IFileHashes => { + try { + return JSON.parse(localStorage.getItem(CACHE_HASHES_KEY) ?? "{}"); + } catch { + return {}; + } +}; + +const setFileHashes = (hashes: IFileHashes) => { + localStorage.setItem(CACHE_HASHES_KEY, JSON.stringify(hashes)); +}; + export class GameCache { private readonly logger: Logger = new Logger("Cache"); private readonly fs: FileSystemManager = new FileSystemManager("game"); async updateCache(manifest: IManifest, force = false): Promise { this.logger.info("Starting cache game files"); - - let extendedManifest = await this._tryReuseCache(manifest, force); - - if (!extendedManifest) { - extendedManifest = await withCacheLock(async () => { - return ( - (await this._tryReuseCache(manifest, force)) ?? (await this._updateCacheProcess(manifest)) - ); - }); - } - - setVersion(manifest.version); + const extendedManifest = await withCacheLock(() => + this._updateCacheProcess(manifest.files, force), + ); this.logger.info("Game files cached"); return extendedManifest; } - private async _tryReuseCache( - manifest: IManifest, + private async _updateCacheProcess( + files: IManifestFile[], force: boolean, - ): Promise { - if (force || !isManifestUpToDate(manifest)) return undefined; - return this._parseCache(manifest); - } + ): Promise { + const stored = force ? {} : getFileHashes(); - private async _updateCacheProcess(manifest: IManifest): Promise { - setLoadingStatus("Cleaning cache game files"); - const directory = await this.fs.getDirectory(); - await directory.clear(); - return { - files: await this._updateCacheFiles(manifest.files), - }; - } + if (!Object.keys(stored).length) { + setLoadingStatus("Cleaning cache game files"); + setFileHashes(stored); + await (await this.fs.getDirectory()).clear(); + } - private async _updateCacheFiles(files: IManifestFile[]): Promise { - const res = []; - setLoadingTotalFiles(files.length); + const remotePaths = new Set(files.map(({ path }) => path)); + for (const path of Object.keys(stored)) { + if (!remotePaths.has(path)) await this._removeCacheFile(path); + } + const hashes: IFileHashes = Object.fromEntries( + Object.entries(stored).filter(([path]) => remotePaths.has(path)), + ); + setFileHashes(hashes); + + setLoadingStatus("Verifying application integrity"); + const res: IExtendedManifestFile[] = []; + const outdated: number[] = []; for (const [i, file] of files.entries()) { - setLoadingStatus(`Download: ${file.path.replace(/^\/+/, "")}`, i); - res.push(await this._updateCacheFile(file)); + const cached = await this._parseFile(file, hashes); + if (cached) res[i] = cached; + else outdated.push(i); } - return res; + + setLoadingTotalFiles(outdated.length); + for (const [n, i] of outdated.entries()) { + const file = files[i] as IManifestFile; + setLoadingStatus( + `Download ${n + 1} / ${outdated.length}: ${file.path.replace(/^\/+/, "")}`, + n, + ); + res[i] = await this._updateCacheFile(file); + hashes[file.path] = file.hash; + setFileHashes(hashes); + } + + return { files: res }; } private async _updateCacheFile(fileManifest: IManifestFile): Promise { @@ -97,35 +119,25 @@ export class GameCache { }; } - private async _parseCache(manifest: IManifest): Promise { - setLoadingStatus("Verifying application integrity"); - return this._parseFiles(manifest.files); - } - - private async _parseFiles(files: IManifestFile[]): Promise { - const res = []; - for (const file of files) { - const r = await this._parseFile(file); - if (!r) return undefined; - res.push(r); - } - return { - files: res, - }; + private async _removeCacheFile(path: string): Promise { + const i = path.lastIndexOf("/"); + await this.fs + .getDirectory(path.slice(0, i)) + .then((directory) => directory.removeChild(path.slice(i + 1))) + .catch(() => undefined); } - /** - * @todo Function to verify if the file passed in parameters exist - * @todo Verify if the file passed in parameters is really complete - */ private async _parseFile( fileManifest: IManifestFile, + hashes: IFileHashes, ): Promise { + if (hashes[fileManifest.path] !== fileManifest.hash) return undefined; try { - const file = await this.fs.getFile(fileManifest.path); + const file = await (await this.fs.getFile(fileManifest.path)).getFile(); + if (file.size !== fileManifest.size) return undefined; return { ...fileManifest, - localPath: await file.getUrl(), + localPath: URL.createObjectURL(file), }; } catch { return undefined; diff --git a/apps/website/src/manifest.ts b/apps/website/src/manifest.ts index 9c3931a..113f3f9 100644 --- a/apps/website/src/manifest.ts +++ b/apps/website/src/manifest.ts @@ -1,5 +1,4 @@ import { type IManifest } from "./types/manifest.type"; -import { getVersion } from "./version"; import { setLoadingStatus } from "./window"; export const getManifest = async (): Promise => { @@ -10,10 +9,3 @@ export const getManifest = async (): Promise => { } return await res.json(); }; - -export const isManifestUpToDate = (manifest: IManifest): boolean => { - setLoadingStatus("Verifying manifest"); - const currentVersion = getVersion(); - if (!currentVersion || currentVersion !== manifest.version) return false; - return true; -}; diff --git a/apps/website/src/types/manifest.type.ts b/apps/website/src/types/manifest.type.ts index 70e60d0..9d1d92f 100644 --- a/apps/website/src/types/manifest.type.ts +++ b/apps/website/src/types/manifest.type.ts @@ -1,5 +1,7 @@ export interface IManifestFile { path: string; + hash: string; + size: number; } export interface IManifest { diff --git a/apps/website/src/version/index.ts b/apps/website/src/version/index.ts deleted file mode 100644 index 5b8dfbe..0000000 --- a/apps/website/src/version/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { clearVersion, getVersion, setVersion } from "./version"; diff --git a/apps/website/src/version/version.ts b/apps/website/src/version/version.ts deleted file mode 100644 index 04de089..0000000 --- a/apps/website/src/version/version.ts +++ /dev/null @@ -1,13 +0,0 @@ -export const GAME_VERSION_SYMBOL = "game-version"; - -export const getVersion = (): string | null => { - return localStorage.getItem(GAME_VERSION_SYMBOL); -}; - -export const setVersion = (version: string) => { - localStorage.setItem(GAME_VERSION_SYMBOL, version); -}; - -export const clearVersion = () => { - localStorage.removeItem(GAME_VERSION_SYMBOL); -}; diff --git a/docs/docs/loader/2-client.mdx b/docs/docs/loader/2-client.mdx index 7a9c4ed..5443bdb 100644 --- a/docs/docs/loader/2-client.mdx +++ b/docs/docs/loader/2-client.mdx @@ -27,7 +27,7 @@ npx loader-client --dir .nanoforge/client --port 3000 | --------------- | --------------------------------------------------------------------- | | `GET /` | Serves the `loader-website` HTML application | | `GET /*` | Serves static assets bundled with `loader-website` | -| `GET /manifest` | Returns the list of game files and the current version as JSON | +| `GET /manifest` | Returns the game files (path, SHA-256 hash, size) and version as JSON | | `GET /env` | Returns `NANOFORGE_*` environment variables (prefix stripped) as JSON | | `GET /game/*` | Serves compiled game client files from the configured game directory | diff --git a/docs/docs/loader/3-website.mdx b/docs/docs/loader/3-website.mdx index 1d0827c..43a4d3b 100644 --- a/docs/docs/loader/3-website.mdx +++ b/docs/docs/loader/3-website.mdx @@ -16,8 +16,8 @@ This package is a dependency of `loader-client` and is not consumed directly. When the browser opens the loader URL, the following steps happen in order: 1. **Fetch manifest** — requests `/manifest` to get the game version and the list of files to download. -2. **Verify cache** — checks whether the game files from the previous session are still present in the browser's Origin Private File System (OPFS). The loading screen shows _"Verifying application integrity"_. -3. **Download files** — if the cache is stale or missing, downloads each game file from `/game/*` and writes it into OPFS. The loading screen shows a progress bar and the name of the current file. +2. **Verify cache** — compares each file's hash and size with the copy cached in the browser's Origin Private File System (OPFS), and removes cached files no longer in the manifest. The loading screen shows _"Verifying application integrity"_. +3. **Download files** — downloads only new or changed files from `/game/*` and writes them into OPFS. The loading screen shows a progress bar and the name of the current file. 4. **Fetch environment** — requests `/env` to get the `NANOFORGE_*` variables forwarded by `loader-client`. 5. **Bootstrap game** — dynamically imports `/main.js` from the local OPFS cache, calls its exported `main({ files, env, container })`, and fades out the loading screen.