Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions src/src/lib/docs/aggregate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import {
resolveDoclinks,
} from './links';
import { stripMkdocsMarkup } from './mkdocs';
import { fetchRawText } from './raw-fetch';
import { parseRedirectFile, REDIRECTS_FILE } from './redirect-file';

export const DOCS_OUTPUT_DIR = resolve('src/content/docs');
Expand Down Expand Up @@ -233,29 +234,28 @@ export async function fetchRawFile(
branch: string,
path: string,
): Promise<RawSourceFile | null> {
const url = `https://raw.githubusercontent.com/${repository}/${branch}/${path}`;
const response = await fetch(url);
if (!response.ok) {
const content = await fetchRawFileContent(repository, branch, path);
if (content === null) {
return null;
}
const content = await response.text();
const lastModified = await lastCommitDate(repository, branch, path);
return { path, content, lastModified };
}

/**
* A repository file's text at a ref, or null when it cannot be read, with no
* commit-date lookup. `fetchRawFile` adds the page's `lastModified` (a second
* A repository file's text at a ref, or null when it does not exist (404), with
* no commit-date lookup. `fetchRawFile` adds the page's `lastModified` (a second
* request to the commit feed); the skill and asset mirrors only need the bytes,
* so they use this and skip it.
* so they use this and skip it. A transient failure is retried and a persistent
* one throws (`fetchRawText`), so a throttled request can never pass for a
* missing file.
*/
export async function fetchRawFileContent(
export function fetchRawFileContent(
repository: string,
ref: string,
path: string,
): Promise<string | null> {
const response = await fetch(`https://raw.githubusercontent.com/${repository}/${ref}/${path}`);
return response.ok ? response.text() : null;
return fetchRawText(`https://raw.githubusercontent.com/${repository}/${ref}/${path}`);
}

/** Reads a repository file for the page being built (a diagram's `.c4` model, ADR 0013). */
Expand Down
112 changes: 112 additions & 0 deletions src/src/lib/docs/raw-fetch.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
/**
* Reading a repository file from `raw.githubusercontent.com`.
*
* The sync asks "does this file exist, and what is in it?", and only a 404 means
* "it does not exist". Every other failure — a 429 from a throttled runner IP, a
* 5xx, a dropped connection — says nothing about the file, so it is retried and,
* if it persists, thrown. Treating those as "absent" used to drop pages from the
* mirror without a word, or fail far from the cause ("no root page exists") when
* the dropped file happened to be the docs root.
*/

export type FetchLike = (input: string, init?: RequestInit) => Promise<Response>;

export interface RawFetchOptions {
fetchImpl?: FetchLike;
/** Total attempts, including the first. */
attempts?: number;
/** Base delay; attempt `n` waits `baseDelayMs * 2^(n-1)` unless `Retry-After` says otherwise. */
baseDelayMs?: number;
/** Upper bound on any single wait, including one requested by `Retry-After`. */
maxDelayMs?: number;
sleep?: (ms: number) => Promise<void>;
}

export class RawFetchError extends Error {
constructor(
readonly url: string,
readonly status: number | null,
detail: string,
) {
super(`raw.githubusercontent.com: ${detail} for ${url}`);
this.name = 'RawFetchError';
}
}

const DEFAULT_ATTEMPTS = 4;
const DEFAULT_BASE_DELAY_MS = 1_000;
const DEFAULT_MAX_DELAY_MS = 30_000;

export function isRetryableStatus(status: number): boolean {
return status === 408 || status === 429 || status >= 500;
}

/** Milliseconds a `Retry-After` header asks for (seconds or an HTTP date), or null. */
export function retryAfterMs(header: string | null, now: number = Date.now()): number | null {
if (!header) {
return null;
}
const trimmed = header.trim();
if (/^\d+$/.test(trimmed)) {
return Number(trimmed) * 1_000;
}
const date = Date.parse(trimmed);
return Number.isNaN(date) ? null : Math.max(0, date - now);
}

const defaultSleep = (ms: number): Promise<void> =>
new Promise((resolve) => setTimeout(resolve, ms));

/**
* The file's text, or null when the server says it does not exist (404).
* Retryable failures are retried with exponential backoff; anything still
* failing afterwards, and any other non-OK status, throws a `RawFetchError`.
*/
export async function fetchRawText(
url: string,
options: RawFetchOptions = {},
): Promise<string | null> {
const fetchImpl = options.fetchImpl ?? fetch;
const attempts = Math.max(1, options.attempts ?? DEFAULT_ATTEMPTS);
const baseDelayMs = options.baseDelayMs ?? DEFAULT_BASE_DELAY_MS;
const maxDelayMs = options.maxDelayMs ?? DEFAULT_MAX_DELAY_MS;
const sleep = options.sleep ?? defaultSleep;

let lastError: RawFetchError | null = null;
for (let attempt = 1; attempt <= attempts; attempt += 1) {
let requestedDelay: number | null = null;
try {
const response = await fetchImpl(url);
if (response.ok) {
return await response.text();
}
if (response.status === 404) {
return null;
}
if (!isRetryableStatus(response.status)) {
throw new RawFetchError(url, response.status, `HTTP ${response.status}`);
}
lastError = new RawFetchError(
url,
response.status,
`HTTP ${response.status} after ${attempt} attempt(s)`,
);
requestedDelay = retryAfterMs(response.headers.get('retry-after'));
} catch (error) {
if (error instanceof RawFetchError) {
throw error;
}
// A network-level failure (reset, DNS, TLS) is as transient as a 5xx.
lastError = new RawFetchError(
url,
null,
`${(error as Error).message} after ${attempt} attempt(s)`,
);
}
if (attempt < attempts) {
const backoff = baseDelayMs * 2 ** (attempt - 1);
await sleep(Math.min(maxDelayMs, requestedDelay ?? backoff));
}
}
throw lastError ?? new RawFetchError(url, null, 'no response');
}
112 changes: 112 additions & 0 deletions src/tests/unit/raw-fetch.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
import type { FetchLike } from '../../src/lib/docs/raw-fetch';

import { describe, expect, test } from 'bun:test';

import {
RawFetchError,
fetchRawText,
isRetryableStatus,
retryAfterMs,
} from '../../src/lib/docs/raw-fetch';

const URL = 'https://raw.githubusercontent.com/purview-dev/containers/main/docs/wiki/Home.md';

/** A fetch that replays the given responses (or throws the given errors) in order. */
function scripted(steps: (Response | Error)[]): { fetchImpl: FetchLike; calls: () => number } {
let calls = 0;
const fetchImpl: FetchLike = async () => {
const step = steps[Math.min(calls, steps.length - 1)];
calls += 1;
if (step instanceof Error) {
throw step;
}
return step!.clone();
};
return { fetchImpl, calls: () => calls };
}

function recordingSleep(): { sleep: (ms: number) => Promise<void>; waits: number[] } {
const waits: number[] = [];
return { sleep: async (ms) => void waits.push(ms), waits };
}

describe('fetchRawText', () => {
test('returns the body of a successful response', async () => {
const { fetchImpl, calls } = scripted([new Response('# Home')]);
expect(await fetchRawText(URL, { fetchImpl })).toBe('# Home');
expect(calls()).toBe(1);
});

test('returns null only for 404, without retrying', async () => {
const { fetchImpl, calls } = scripted([new Response('', { status: 404 })]);
expect(await fetchRawText(URL, { fetchImpl })).toBeNull();
expect(calls()).toBe(1);
});

test('retries a 429 and a 5xx, then returns the file', async () => {
const { sleep, waits } = recordingSleep();
const { fetchImpl, calls } = scripted([
new Response('', { status: 429 }),
new Response('', { status: 503 }),
new Response('# Home'),
]);
expect(await fetchRawText(URL, { fetchImpl, sleep, baseDelayMs: 100 })).toBe('# Home');
expect(calls()).toBe(3);
expect(waits).toEqual([100, 200]);
});

test('retries a network error', async () => {
const { sleep } = recordingSleep();
const { fetchImpl, calls } = scripted([new TypeError('socket hang up'), new Response('ok')]);
expect(await fetchRawText(URL, { fetchImpl, sleep })).toBe('ok');
expect(calls()).toBe(2);
});

test('honours Retry-After, capped at maxDelayMs', async () => {
const { sleep, waits } = recordingSleep();
const { fetchImpl } = scripted([
new Response('', { status: 429, headers: { 'Retry-After': '3' } }),
new Response('', { status: 429, headers: { 'Retry-After': '120' } }),
new Response('ok'),
]);
await fetchRawText(URL, { fetchImpl, sleep, maxDelayMs: 10_000 });
expect(waits).toEqual([3_000, 10_000]);
});

test('throws with the status and URL when retries are exhausted, never null', async () => {
const { sleep, waits } = recordingSleep();
const { fetchImpl, calls } = scripted([new Response('', { status: 429 })]);
const failure = fetchRawText(URL, { fetchImpl, sleep, attempts: 3 });
await expect(failure).rejects.toBeInstanceOf(RawFetchError);
await expect(failure).rejects.toThrow(`HTTP 429 after 3 attempt(s) for ${URL}`);
expect(calls()).toBe(3);
expect(waits).toHaveLength(2);
});

test('throws immediately on a non-retryable status', async () => {
const { fetchImpl, calls } = scripted([new Response('', { status: 403 })]);
await expect(fetchRawText(URL, { fetchImpl })).rejects.toThrow('HTTP 403');
expect(calls()).toBe(1);
});
});

describe('isRetryableStatus', () => {
test('retries throttling, timeouts and server errors only', () => {
expect([408, 429, 500, 502, 503, 504].every(isRetryableStatus)).toBe(true);
expect([400, 401, 403, 404, 410].some(isRetryableStatus)).toBe(false);
});
});

describe('retryAfterMs', () => {
test('reads delta-seconds and HTTP dates', () => {
expect(retryAfterMs('5')).toBe(5_000);
const now = Date.parse('2026-11-19T18:00:00Z');
expect(retryAfterMs('Thu, 19 Nov 2026 18:00:10 GMT', now)).toBe(10_000);
expect(retryAfterMs('Thu, 19 Nov 2026 17:00:00 GMT', now)).toBe(0);
});

test('ignores a missing or unreadable header', () => {
expect(retryAfterMs(null)).toBeNull();
expect(retryAfterMs('soon')).toBeNull();
});
});
Loading