From b00365e29a800bd966994f2456a7c3657f68b13a Mon Sep 17 00:00:00 2001 From: Charles Lyding <19598772+clydin@users.noreply.github.com> Date: Thu, 3 Sep 2026 10:28:41 -0400 Subject: [PATCH] fix(@angular/build): add automatic corruption recovery in SQLite cache store Abrupt process terminations (such as canceling a watch mode build or CI runner timeouts) can leave SQLite databases or their write-ahead logs in an inconsistent or corrupted state. Previously, corrupted database files or malformed pages caused subsequent builds to fail continuously until the cache directory was manually deleted. Additionally, uninitializable cache stores on read-only or restricted filesystems would throw errors during build initialization. Corrupted cache files (.db, .db-wal, and .db-shm) are now automatically removed and recreated upon initialization failure. If the cache cannot be initialized or recovered (such as on read-only filesystems or due to permission errors), the cache store gracefully disables itself. Cache operations subsequently degrade cleanly to cache misses, allowing builds to proceed successfully without error. --- .../src/tools/esbuild/sqlite-cache-store.ts | 196 +++++++++++++----- .../tools/esbuild/sqlite-cache-store_spec.ts | 131 ++++++++++++ 2 files changed, 278 insertions(+), 49 deletions(-) diff --git a/packages/angular/build/src/tools/esbuild/sqlite-cache-store.ts b/packages/angular/build/src/tools/esbuild/sqlite-cache-store.ts index 9de037800844..981bafb9db74 100644 --- a/packages/angular/build/src/tools/esbuild/sqlite-cache-store.ts +++ b/packages/angular/build/src/tools/esbuild/sqlite-cache-store.ts @@ -6,12 +6,35 @@ * found in the LICENSE file at https://angular.dev/license */ -import { mkdirSync } from 'node:fs'; +import { mkdirSync, rmSync } from 'node:fs'; import { dirname } from 'node:path'; import { DatabaseSync, StatementSync } from 'node:sqlite'; import { deserialize, serialize } from 'node:v8'; import { Cache, PersistentCacheStore } from './cache'; +/** + * Common SQLite primary result codes. + * @see https://www.sqlite.org/rescode.html + */ +const enum SqliteResultCode { + Busy = 5, + Locked = 6, +} + +interface SqliteError extends Error { + code?: string; + errcode?: number; + errstr?: string; +} + +function isSqliteError(error: unknown): error is SqliteError { + return ( + error instanceof Error && + ('errcode' in error || + ('code' in error && (error as { code: unknown }).code === 'ERR_SQLITE_ERROR')) + ); +} + /** * A persistent cache store backed by SQLite. * @@ -22,56 +45,114 @@ import { Cache, PersistentCacheStore } from './cache'; */ export class SqliteCacheStore implements PersistentCacheStore { #db: DatabaseSync | undefined; + #disabled = false; #getStmt: StatementSync | undefined; #hasStmt: StatementSync | undefined; #setStmt: StatementSync | undefined; #updateAccessedStmt: StatementSync | undefined; readonly #pendingAccessedKeys = new Set(); #flushTimeout: NodeJS.Timeout | undefined; + readonly #busyTimeoutMs: number; constructor( readonly cachePath: string, private readonly maxPayloadSize = 1024 * 1024 * 1024, private readonly ttlDays = 14, - ) {} + busyTimeoutMs = 5000, + ) { + this.#busyTimeoutMs = + Number.isSafeInteger(busyTimeoutMs) && busyTimeoutMs >= 0 ? busyTimeoutMs : 5000; + } - #ensureDb(): DatabaseSync { - if (!this.#db) { + #openDatabase(): DatabaseSync { + let db: DatabaseSync | undefined; + try { if (this.cachePath === ':memory:') { - this.#db = new DatabaseSync(this.cachePath); + db = new DatabaseSync(this.cachePath); } else { // Optimistically attempt to open the database file first to avoid directory creation // syscalls on warm builds where the parent directory already exists. try { - this.#db = new DatabaseSync(this.cachePath); + db = new DatabaseSync(this.cachePath); } catch { mkdirSync(dirname(this.cachePath), { recursive: true }); - this.#db = new DatabaseSync(this.cachePath); + db = new DatabaseSync(this.cachePath); } } // Optimize SQLite for cache usage - this.#db.exec('PRAGMA auto_vacuum = FULL;'); - this.#db.exec('PRAGMA journal_mode = WAL;'); - this.#db.exec('PRAGMA synchronous = NORMAL;'); - this.#db.exec('PRAGMA busy_timeout = 5000;'); - this.#db.exec('PRAGMA temp_store = MEMORY;'); - this.#db.exec('PRAGMA mmap_size = 268435456;'); - this.#db.exec( + db.exec(`PRAGMA busy_timeout = ${this.#busyTimeoutMs};`); + db.exec('PRAGMA auto_vacuum = FULL;'); + db.exec('PRAGMA journal_mode = WAL;'); + db.exec('PRAGMA synchronous = NORMAL;'); + db.exec('PRAGMA temp_store = MEMORY;'); + db.exec('PRAGMA mmap_size = 268435456;'); + db.exec( 'CREATE TABLE IF NOT EXISTS cache (key TEXT PRIMARY KEY, value BLOB, last_accessed INTEGER NOT NULL) WITHOUT ROWID;', ); - this.#db.exec( + db.exec( 'CREATE INDEX IF NOT EXISTS idx_cache_accessed ON cache (last_accessed DESC, key DESC);', ); - this.#getStmt = this.#db.prepare('SELECT value FROM cache WHERE key = ?'); - this.#hasStmt = this.#db.prepare('SELECT 1 FROM cache WHERE key = ?'); - this.#setStmt = this.#db.prepare( + this.#getStmt = db.prepare('SELECT value FROM cache WHERE key = ?'); + this.#hasStmt = db.prepare('SELECT 1 FROM cache WHERE key = ?'); + this.#setStmt = db.prepare( 'INSERT OR REPLACE INTO cache (key, value, last_accessed) VALUES (?, ?, unixepoch())', ); - this.#updateAccessedStmt = this.#db.prepare( + this.#updateAccessedStmt = db.prepare( 'UPDATE cache SET last_accessed = unixepoch() WHERE key = ?', ); + + this.#db = db; + + return db; + } catch (error) { + try { + db?.close(); + } catch { + // Ignore close error on corrupted handle + } + this.#getStmt = undefined; + this.#hasStmt = undefined; + this.#setStmt = undefined; + this.#updateAccessedStmt = undefined; + throw error; + } + } + + #ensureDb(): DatabaseSync | undefined { + if (this.#disabled) { + return undefined; + } + + if (!this.#db) { + try { + return this.#openDatabase(); + } catch (error) { + // If the database is locked by another active process, + // do not attempt to delete the database files as that could corrupt the active process's database. + const isBusy = + isSqliteError(error) && + (error.errcode === SqliteResultCode.Busy || error.errcode === SqliteResultCode.Locked); + + // Attempt to recover from database corruption by deleting the corrupted files and recreating + if (!isBusy && this.cachePath !== ':memory:') { + try { + rmSync(this.cachePath, { force: true }); + rmSync(this.cachePath + '-wal', { force: true }); + rmSync(this.cachePath + '-shm', { force: true }); + rmSync(this.cachePath + '-journal', { force: true }); + + return this.#openDatabase(); + } catch { + // If recovery fails (e.g. read-only filesystem or permission denied), disable caching + } + } + + this.#disabled = true; + + return undefined; + } } return this.#db; @@ -94,19 +175,21 @@ export class SqliteCacheStore implements PersistentCacheStore { this.#flushTimeout = undefined; } - if (!this.#db || this.#pendingAccessedKeys.size === 0 || !this.#updateAccessedStmt) { + if (this.#pendingAccessedKeys.size === 0) { return; } try { - this.#db.exec('BEGIN IMMEDIATE TRANSACTION;'); - for (const key of this.#pendingAccessedKeys) { - this.#updateAccessedStmt.run(key); + if (this.#db && this.#updateAccessedStmt) { + this.#db.exec('BEGIN IMMEDIATE TRANSACTION;'); + for (const key of this.#pendingAccessedKeys) { + this.#updateAccessedStmt.run(key); + } + this.#db.exec('COMMIT;'); } - this.#db.exec('COMMIT;'); } catch { try { - this.#db.exec('ROLLBACK;'); + this.#db?.exec('ROLLBACK;'); } catch { // Ignore rollback errors if transaction was not active } @@ -117,35 +200,55 @@ export class SqliteCacheStore implements PersistentCacheStore { // eslint-disable-next-line @typescript-eslint/no-explicit-any async get(key: string): Promise { - this.#ensureDb(); - // SQLite column types are dynamic, so the stored value is only known at runtime. - const row = this.#getStmt?.get(key) as { value: unknown } | undefined; + if (!this.#ensureDb()) { + return undefined; + } + + try { + // SQLite column types are dynamic, so the stored value is only known at runtime. + const row = this.#getStmt?.get(key) as { value: unknown } | undefined; - if (row) { - this.#queueAccessUpdate(key); + if (row) { + this.#queueAccessUpdate(key); - if (row.value instanceof Uint8Array) { - try { - return deserialize(row.value); - } catch { - // Treat corrupt or unparseable cached payloads as a cache miss. + if (row.value instanceof Uint8Array) { + try { + return deserialize(row.value); + } catch { + // Treat corrupt or unparseable cached payloads as a cache miss. + } } } + } catch { + // Treat query errors (e.g. disk read failures) as a cache miss. } return undefined; } has(key: string): boolean { - this.#ensureDb(); + if (!this.#ensureDb()) { + return false; + } - return !!this.#hasStmt?.get(key); + try { + return !!this.#hasStmt?.get(key); + } catch { + return false; + } } async set(key: string, value: unknown): Promise { - this.#ensureDb(); - this.#pendingAccessedKeys.delete(key); - this.#setStmt?.run(key, serialize(value)); + if (!this.#ensureDb()) { + return this; + } + + try { + this.#pendingAccessedKeys.delete(key); + this.#setStmt?.run(key, serialize(value)); + } catch { + // Writing to cache is non-fatal and should not fail the build. + } return this; } @@ -155,11 +258,10 @@ export class SqliteCacheStore implements PersistentCacheStore { } close(): void { + this.#flushAccessUpdates(); + if (this.#db) { try { - // Flush any pending access updates in one transaction before pruning - this.#flushAccessUpdates(); - this.#db.exec('BEGIN IMMEDIATE TRANSACTION;'); try { // 1. Delete items older than N days @@ -202,12 +304,6 @@ export class SqliteCacheStore implements PersistentCacheStore { } catch { // Pruning errors should not block build success } finally { - if (this.#flushTimeout) { - clearTimeout(this.#flushTimeout); - this.#flushTimeout = undefined; - } - this.#pendingAccessedKeys.clear(); - this.#getStmt = undefined; this.#hasStmt = undefined; this.#setStmt = undefined; @@ -221,5 +317,7 @@ export class SqliteCacheStore implements PersistentCacheStore { this.#db = undefined; } } + + this.#disabled = false; } } diff --git a/packages/angular/build/src/tools/esbuild/sqlite-cache-store_spec.ts b/packages/angular/build/src/tools/esbuild/sqlite-cache-store_spec.ts index 15eec7cb1298..f5b9ccd1960d 100644 --- a/packages/angular/build/src/tools/esbuild/sqlite-cache-store_spec.ts +++ b/packages/angular/build/src/tools/esbuild/sqlite-cache-store_spec.ts @@ -93,6 +93,137 @@ describe('SqliteCacheStore', () => { } }); + it('should automatically recover from a corrupted database file', async () => { + // Write corrupted header data to the database file + await fs.writeFile(cachePath, 'CORRUPTED_DATABASE_FILE_CONTENTS'); + + // The store should detect corruption, reset the database files, and operate normally + await store.set('recover-key', 'recovered-value'); + const result = await store.get('recover-key'); + expect(result).toBe('recovered-value'); + }); + + it('should automatically recover from a corrupted B-tree page', async () => { + // Initialize valid database + await store.set('initial-key', 'initial-val'); + store.close(); + + // Overwrite page data with garbage + const handle = await fs.open(cachePath, 'r+'); + await handle.write(Buffer.alloc(200, 0xff), 0, 200, 100); + await handle.close(); + + const reopenedStore = new SqliteCacheStore(cachePath); + try { + await reopenedStore.set('new-key', 'new-val'); + expect(await reopenedStore.get('new-key')).toBe('new-val'); + } finally { + reopenedStore.close(); + } + }); + + it('should gracefully degrade when database path is permanently unwritable', async () => { + // A regular file in place of a directory path ensures unwritability across all platforms (ENOTDIR) + const blockingFile = join(tempDir, 'blocking-file'); + await fs.writeFile(blockingFile, 'cannot-be-a-directory'); + + const unwritableStore = new SqliteCacheStore(join(blockingFile, 'cannot-create.db')); + try { + expect(unwritableStore.has('any-key')).toBeFalse(); + expect(await unwritableStore.get('any-key')).toBeUndefined(); + await unwritableStore.set('any-key', 'any-val'); + expect(await unwritableStore.get('any-key')).toBeUndefined(); + expect(unwritableStore.has('any-key')).toBeFalse(); + } finally { + unwritableStore.close(); + } + }); + + it('should automatically recover from a corrupted journal file', async () => { + await store.set('key-before', 'val-before'); + store.close(); + + // Create a corrupt rollback journal file + await fs.writeFile(cachePath + '-journal', 'CORRUPTED_JOURNAL_FILE'); + + const reopenedStore = new SqliteCacheStore(cachePath); + try { + await reopenedStore.set('new-key', 'new-val'); + expect(await reopenedStore.get('new-key')).toBe('new-val'); + } finally { + reopenedStore.close(); + } + }); + + it('should not delete database files when database is locked by another process', async () => { + await store.set('persist-key', 'persist-val'); + store.close(); + + // Open direct connection with an exclusive transaction holding a write lock + const { DatabaseSync } = await import('node:sqlite'); + const directDb = new DatabaseSync(cachePath); + directDb.exec('BEGIN EXCLUSIVE TRANSACTION;'); + + // Use a tiny busyTimeoutMs so the test fails fast instead of waiting 5s + const lockedStore = new SqliteCacheStore(cachePath, undefined, undefined, 10); + try { + expect(lockedStore.has('persist-key')).toBeFalse(); + expect(await lockedStore.get('persist-key')).toBeUndefined(); + } finally { + lockedStore.close(); + directDb.exec('COMMIT;'); + directDb.close(); + } + + // Verify the original database file and its data were not deleted + const verifyStore = new SqliteCacheStore(cachePath); + try { + expect(await verifyStore.get('persist-key')).toBe('persist-val'); + } finally { + verifyStore.close(); + } + }); + + it('should safely fall back to default timeout if invalid busyTimeoutMs is provided', async () => { + // Pass NaN as busyTimeoutMs + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const invalidStore = new SqliteCacheStore(cachePath, undefined, undefined, NaN as any); + try { + await invalidStore.set('valid-key', 'valid-value'); + expect(await invalidStore.get('valid-key')).toBe('valid-value'); + } finally { + invalidStore.close(); + } + }); + + it('should flush pending access updates on close', async () => { + await store.set('flush-key', 'flush-val'); + + // Manually backdate the entry's last_accessed timestamp to simulate elapsed time + const { DatabaseSync } = await import('node:sqlite'); + const directDb = new DatabaseSync(cachePath); + const pastTimestamp = Math.floor(Date.now() / 1000) - 3600; + directDb + .prepare('UPDATE cache SET last_accessed = ? WHERE key = ?') + .run(pastTimestamp, 'flush-key'); + directDb.close(); + + // Access the key via store.get() to queue an access update + await store.get('flush-key'); + + // Immediately close the store before the debounced 500ms timeout fires + store.close(); + + // Verify the timestamp in SQLite was updated upon close + const checkDb = new DatabaseSync(cachePath); + const row = checkDb + .prepare('SELECT last_accessed FROM cache WHERE key = ?') + .get('flush-key') as { last_accessed: number }; + checkDb.close(); + + expect(row.last_accessed).toBeGreaterThan(pastTimestamp); + }); + it('should treat a non-binary payload as a cache miss', async () => { await store.set('text-key', 'value'); store.close();