diff --git a/src/src/lib/docs/aggregate.ts b/src/src/lib/docs/aggregate.ts index 65062d2..2d40569 100644 --- a/src/src/lib/docs/aggregate.ts +++ b/src/src/lib/docs/aggregate.ts @@ -35,6 +35,7 @@ import { resolveDoclinks, } from './links'; import { stripMkdocsMarkup } from './mkdocs'; +import { fetchRawText } from './raw-fetch'; import { parseRedirectFile, REDIRECTS_FILE } from './redirect-file'; export const DOCS_OUTPUT_DIR = resolve('src/content/docs'); @@ -233,29 +234,28 @@ export async function fetchRawFile( branch: string, path: string, ): Promise { - const url = `https://raw.githubusercontent.com/${repository}/${branch}/${path}`; - const response = await fetch(url); - if (!response.ok) { + const content = await fetchRawFileContent(repository, branch, path); + if (content === null) { return null; } - const content = await response.text(); const lastModified = await lastCommitDate(repository, branch, path); return { path, content, lastModified }; } /** - * A repository file's text at a ref, or null when it cannot be read, with no - * commit-date lookup. `fetchRawFile` adds the page's `lastModified` (a second + * A repository file's text at a ref, or null when it does not exist (404), with + * no commit-date lookup. `fetchRawFile` adds the page's `lastModified` (a second * request to the commit feed); the skill and asset mirrors only need the bytes, - * so they use this and skip it. + * so they use this and skip it. A transient failure is retried and a persistent + * one throws (`fetchRawText`), so a throttled request can never pass for a + * missing file. */ -export async function fetchRawFileContent( +export function fetchRawFileContent( repository: string, ref: string, path: string, ): Promise { - const response = await fetch(`https://raw.githubusercontent.com/${repository}/${ref}/${path}`); - return response.ok ? response.text() : null; + return fetchRawText(`https://raw.githubusercontent.com/${repository}/${ref}/${path}`); } /** Reads a repository file for the page being built (a diagram's `.c4` model, ADR 0013). */ diff --git a/src/src/lib/docs/raw-fetch.ts b/src/src/lib/docs/raw-fetch.ts new file mode 100644 index 0000000..e5be5bc --- /dev/null +++ b/src/src/lib/docs/raw-fetch.ts @@ -0,0 +1,112 @@ +/** + * Reading a repository file from `raw.githubusercontent.com`. + * + * The sync asks "does this file exist, and what is in it?", and only a 404 means + * "it does not exist". Every other failure — a 429 from a throttled runner IP, a + * 5xx, a dropped connection — says nothing about the file, so it is retried and, + * if it persists, thrown. Treating those as "absent" used to drop pages from the + * mirror without a word, or fail far from the cause ("no root page exists") when + * the dropped file happened to be the docs root. + */ + +export type FetchLike = (input: string, init?: RequestInit) => Promise; + +export interface RawFetchOptions { + fetchImpl?: FetchLike; + /** Total attempts, including the first. */ + attempts?: number; + /** Base delay; attempt `n` waits `baseDelayMs * 2^(n-1)` unless `Retry-After` says otherwise. */ + baseDelayMs?: number; + /** Upper bound on any single wait, including one requested by `Retry-After`. */ + maxDelayMs?: number; + sleep?: (ms: number) => Promise; +} + +export class RawFetchError extends Error { + constructor( + readonly url: string, + readonly status: number | null, + detail: string, + ) { + super(`raw.githubusercontent.com: ${detail} for ${url}`); + this.name = 'RawFetchError'; + } +} + +const DEFAULT_ATTEMPTS = 4; +const DEFAULT_BASE_DELAY_MS = 1_000; +const DEFAULT_MAX_DELAY_MS = 30_000; + +export function isRetryableStatus(status: number): boolean { + return status === 408 || status === 429 || status >= 500; +} + +/** Milliseconds a `Retry-After` header asks for (seconds or an HTTP date), or null. */ +export function retryAfterMs(header: string | null, now: number = Date.now()): number | null { + if (!header) { + return null; + } + const trimmed = header.trim(); + if (/^\d+$/.test(trimmed)) { + return Number(trimmed) * 1_000; + } + const date = Date.parse(trimmed); + return Number.isNaN(date) ? null : Math.max(0, date - now); +} + +const defaultSleep = (ms: number): Promise => + new Promise((resolve) => setTimeout(resolve, ms)); + +/** + * The file's text, or null when the server says it does not exist (404). + * Retryable failures are retried with exponential backoff; anything still + * failing afterwards, and any other non-OK status, throws a `RawFetchError`. + */ +export async function fetchRawText( + url: string, + options: RawFetchOptions = {}, +): Promise { + const fetchImpl = options.fetchImpl ?? fetch; + const attempts = Math.max(1, options.attempts ?? DEFAULT_ATTEMPTS); + const baseDelayMs = options.baseDelayMs ?? DEFAULT_BASE_DELAY_MS; + const maxDelayMs = options.maxDelayMs ?? DEFAULT_MAX_DELAY_MS; + const sleep = options.sleep ?? defaultSleep; + + let lastError: RawFetchError | null = null; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + let requestedDelay: number | null = null; + try { + const response = await fetchImpl(url); + if (response.ok) { + return await response.text(); + } + if (response.status === 404) { + return null; + } + if (!isRetryableStatus(response.status)) { + throw new RawFetchError(url, response.status, `HTTP ${response.status}`); + } + lastError = new RawFetchError( + url, + response.status, + `HTTP ${response.status} after ${attempt} attempt(s)`, + ); + requestedDelay = retryAfterMs(response.headers.get('retry-after')); + } catch (error) { + if (error instanceof RawFetchError) { + throw error; + } + // A network-level failure (reset, DNS, TLS) is as transient as a 5xx. + lastError = new RawFetchError( + url, + null, + `${(error as Error).message} after ${attempt} attempt(s)`, + ); + } + if (attempt < attempts) { + const backoff = baseDelayMs * 2 ** (attempt - 1); + await sleep(Math.min(maxDelayMs, requestedDelay ?? backoff)); + } + } + throw lastError ?? new RawFetchError(url, null, 'no response'); +} diff --git a/src/tests/unit/raw-fetch.test.ts b/src/tests/unit/raw-fetch.test.ts new file mode 100644 index 0000000..d2a2380 --- /dev/null +++ b/src/tests/unit/raw-fetch.test.ts @@ -0,0 +1,112 @@ +import type { FetchLike } from '../../src/lib/docs/raw-fetch'; + +import { describe, expect, test } from 'bun:test'; + +import { + RawFetchError, + fetchRawText, + isRetryableStatus, + retryAfterMs, +} from '../../src/lib/docs/raw-fetch'; + +const URL = 'https://raw.githubusercontent.com/purview-dev/containers/main/docs/wiki/Home.md'; + +/** A fetch that replays the given responses (or throws the given errors) in order. */ +function scripted(steps: (Response | Error)[]): { fetchImpl: FetchLike; calls: () => number } { + let calls = 0; + const fetchImpl: FetchLike = async () => { + const step = steps[Math.min(calls, steps.length - 1)]; + calls += 1; + if (step instanceof Error) { + throw step; + } + return step!.clone(); + }; + return { fetchImpl, calls: () => calls }; +} + +function recordingSleep(): { sleep: (ms: number) => Promise; waits: number[] } { + const waits: number[] = []; + return { sleep: async (ms) => void waits.push(ms), waits }; +} + +describe('fetchRawText', () => { + test('returns the body of a successful response', async () => { + const { fetchImpl, calls } = scripted([new Response('# Home')]); + expect(await fetchRawText(URL, { fetchImpl })).toBe('# Home'); + expect(calls()).toBe(1); + }); + + test('returns null only for 404, without retrying', async () => { + const { fetchImpl, calls } = scripted([new Response('', { status: 404 })]); + expect(await fetchRawText(URL, { fetchImpl })).toBeNull(); + expect(calls()).toBe(1); + }); + + test('retries a 429 and a 5xx, then returns the file', async () => { + const { sleep, waits } = recordingSleep(); + const { fetchImpl, calls } = scripted([ + new Response('', { status: 429 }), + new Response('', { status: 503 }), + new Response('# Home'), + ]); + expect(await fetchRawText(URL, { fetchImpl, sleep, baseDelayMs: 100 })).toBe('# Home'); + expect(calls()).toBe(3); + expect(waits).toEqual([100, 200]); + }); + + test('retries a network error', async () => { + const { sleep } = recordingSleep(); + const { fetchImpl, calls } = scripted([new TypeError('socket hang up'), new Response('ok')]); + expect(await fetchRawText(URL, { fetchImpl, sleep })).toBe('ok'); + expect(calls()).toBe(2); + }); + + test('honours Retry-After, capped at maxDelayMs', async () => { + const { sleep, waits } = recordingSleep(); + const { fetchImpl } = scripted([ + new Response('', { status: 429, headers: { 'Retry-After': '3' } }), + new Response('', { status: 429, headers: { 'Retry-After': '120' } }), + new Response('ok'), + ]); + await fetchRawText(URL, { fetchImpl, sleep, maxDelayMs: 10_000 }); + expect(waits).toEqual([3_000, 10_000]); + }); + + test('throws with the status and URL when retries are exhausted, never null', async () => { + const { sleep, waits } = recordingSleep(); + const { fetchImpl, calls } = scripted([new Response('', { status: 429 })]); + const failure = fetchRawText(URL, { fetchImpl, sleep, attempts: 3 }); + await expect(failure).rejects.toBeInstanceOf(RawFetchError); + await expect(failure).rejects.toThrow(`HTTP 429 after 3 attempt(s) for ${URL}`); + expect(calls()).toBe(3); + expect(waits).toHaveLength(2); + }); + + test('throws immediately on a non-retryable status', async () => { + const { fetchImpl, calls } = scripted([new Response('', { status: 403 })]); + await expect(fetchRawText(URL, { fetchImpl })).rejects.toThrow('HTTP 403'); + expect(calls()).toBe(1); + }); +}); + +describe('isRetryableStatus', () => { + test('retries throttling, timeouts and server errors only', () => { + expect([408, 429, 500, 502, 503, 504].every(isRetryableStatus)).toBe(true); + expect([400, 401, 403, 404, 410].some(isRetryableStatus)).toBe(false); + }); +}); + +describe('retryAfterMs', () => { + test('reads delta-seconds and HTTP dates', () => { + expect(retryAfterMs('5')).toBe(5_000); + const now = Date.parse('2026-11-19T18:00:00Z'); + expect(retryAfterMs('Thu, 19 Nov 2026 18:00:10 GMT', now)).toBe(10_000); + expect(retryAfterMs('Thu, 19 Nov 2026 17:00:00 GMT', now)).toBe(0); + }); + + test('ignores a missing or unreadable header', () => { + expect(retryAfterMs(null)).toBeNull(); + expect(retryAfterMs('soon')).toBeNull(); + }); +});