From cc1bea4ff003a9b2c4f50938277796993bce8861 Mon Sep 17 00:00:00 2001 From: Amr Mohammed El-Sheraey <141947355+AmrMsCLL@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:25:37 +0200 Subject: [PATCH] feat: alert on the hub's error rate and failing readiness --- app/src/server/alerts.ts | 2 ++ app/src/server/config.ts | 1 + app/src/server/hub.ts | 72 +++++++++++++++++++++++++++++++++++++++ app/src/server/monitor.ts | 2 ++ app/test/hub.spec.ts | 47 +++++++++++++++++++++++++ 5 files changed, 124 insertions(+) create mode 100644 app/src/server/hub.ts create mode 100644 app/test/hub.spec.ts diff --git a/app/src/server/alerts.ts b/app/src/server/alerts.ts index 3e11927..4ab704e 100644 --- a/app/src/server/alerts.ts +++ b/app/src/server/alerts.ts @@ -27,6 +27,8 @@ export const ruleKinds = { 'container-unhealthy': { label: 'A container unhealthy', unit: null, threshold: 0, minutes: 1 }, 'container-restarts': { label: 'A container restarted in 10 minutes, times', unit: 'restarts', threshold: 3, minutes: 0 }, 'backup-late': { label: 'A project without a backup for', unit: 'hours', threshold: 26, minutes: 0 }, + 'hub-errors': { label: 'Hub requests failing over five minutes above', unit: '%', threshold: 1, minutes: 0 }, + 'hub-ready': { label: 'Hub readiness failing, checks in a row', unit: 'checks', threshold: 2, minutes: 0 }, } as const; export type RuleKind = keyof typeof ruleKinds; diff --git a/app/src/server/config.ts b/app/src/server/config.ts index 2ff125a..b67a9b1 100644 --- a/app/src/server/config.ts +++ b/app/src/server/config.ts @@ -33,6 +33,7 @@ export const config = { self: process.env.HOSTNAME ?? '', sshTarget: process.env.SSH_TARGET ?? 'ubuntu@localhost', secretsKey: process.env.SECRETS_KEY || undefined, + deployReportSecret: process.env.DEPLOY_REPORT_SECRET || undefined, graphiteHome: process.env.GRAPHITE_HOME ?? '/srv/graphite', platformRepo: process.env.PLATFORM_REPO ?? 'project-graphite/platform', platformApp: diff --git a/app/src/server/hub.ts b/app/src/server/hub.ts new file mode 100644 index 0000000..cc85a6c --- /dev/null +++ b/app/src/server/hub.ts @@ -0,0 +1,72 @@ +import { createHmac } from 'node:crypto'; +import { defaultRules, type Condition, type Rule, type RuleKind } from './alerts.ts'; +import { config } from './config.ts'; +import type { ContainerView } from './inventory.ts'; + +export interface HubMetrics { + windowMinutes: number; + requests: number; + errorRate: number; + p95Ms: number | null; + liveConnections: number; +} + +type Rules = Record; + +const HUB = 'http://graphite-hub:3000'; + +let failedChecks = 0; + +export function hubRunning(views: ContainerView[]) { + return views.some((container) => container.project === 'graphite-hub' && container.service === 'app' && container.state === 'running'); +} + +export function hubConditions(failures: number, metrics: HubMetrics | undefined, rules: Rules = defaultRules): Condition[] { + const percent = metrics ? 100 * metrics.errorRate : 0; + return [ + ...(rules['hub-ready'].enabled && failures >= rules['hub-ready'].threshold + ? [{ key: 'hub:ready', title: 'The hub is not ready', detail: `/health/ready failed ${failures} checks in a row.` }] + : []), + ...(metrics && rules['hub-errors'].enabled && percent > rules['hub-errors'].threshold + ? [ + { + key: 'hub:errors', + title: `The hub answers ${percent.toFixed(1)}% of requests with errors`, + detail: `Over ${rules['hub-errors'].threshold}% of ${metrics.requests} requests in the last ${metrics.windowMinutes} minutes; p95 ${metrics.p95Ms ?? 0} ms.`, + for: rules['hub-errors'].minutes * 60_000, + }, + ] + : []), + ]; +} + +export function signedHeaders(secret: string, now = Date.now()) { + const stamp = String(Math.floor(now / 1000)); + return { 'X-Graphite-Timestamp': stamp, 'X-Graphite-Signature': createHmac('sha256', secret).update(`${stamp}.`).digest('hex') }; +} + +export async function checkHub(rules: Rules) { + const secret = config.deployReportSecret; + if (!secret) { + return []; + } + const ready = await fetch(`${HUB}/health/ready`, { signal: AbortSignal.timeout(5_000) }).then( + (response) => response.ok, + () => false, + ); + failedChecks = ready ? 0 : failedChecks + 1; + if (!ready) { + return hubConditions(failedChecks, undefined, rules); + } + const response = await fetch(`${HUB}/internal/metrics`, { headers: signedHeaders(secret), signal: AbortSignal.timeout(5_000) }); + if (!response.ok) { + return [ + { + key: 'hub:metrics', + title: 'The console cannot read the hub metrics', + detail: `The hub answered ${response.status}; DEPLOY_REPORT_SECRET must match in the console and hub env files.`, + }, + ]; + } + return hubConditions(failedChecks, (await response.json()) as HubMetrics, rules); +} diff --git a/app/src/server/monitor.ts b/app/src/server/monitor.ts index 9d30874..5e3e5de 100644 --- a/app/src/server/monitor.ts +++ b/app/src/server/monitor.ts @@ -3,6 +3,7 @@ import { defaultRules, type Alert, type Alerts, type Condition, type Rule, type import { backupConditions } from './backups.ts'; import { config } from './config.ts'; import { docker } from './docker.ts'; +import { checkHub, hubRunning } from './hub.ts'; import { snapshot, type ContainerView } from './inventory.ts'; import { latest, type Frame, type HostLive } from './live.ts'; import { Refusal } from './refusal.ts'; @@ -109,6 +110,7 @@ async function conditions(rules: Rules, now: number) { ...containerConditions(views, inspects, now, rules), ...usageConditions(frame, rules), ...(rules['backup-late'].enabled ? await backupConditions(now, undefined, rules['backup-late'].threshold) : []), + ...(hubRunning(views) ? await checkHub(rules) : []), ]; } diff --git a/app/test/hub.spec.ts b/app/test/hub.spec.ts new file mode 100644 index 0000000..1bfbcc1 --- /dev/null +++ b/app/test/hub.spec.ts @@ -0,0 +1,47 @@ +import { createHmac } from 'node:crypto'; +import { describe, expect, it } from 'vitest'; +import { defaultRules } from '../src/server/alerts.ts'; +import { hubConditions, hubRunning, signedHeaders, type HubMetrics } from '../src/server/hub.ts'; +import type { ContainerView } from '../src/server/inventory.ts'; + +const metrics = (errorRate: number): HubMetrics => ({ windowMinutes: 5, requests: 400, errorRate, p95Ms: 42, liveConnections: 3 }); + +describe('hubConditions', () => { + it('alerts when more than one percent of requests fail over the window', () => { + expect(hubConditions(0, metrics(0.01))).toEqual([]); + expect(hubConditions(0, metrics(0.025))).toEqual([ + expect.objectContaining({ key: 'hub:errors', title: 'The hub answers 2.5% of requests with errors' }), + ]); + }); + + it('alerts when readiness fails twice in a row', () => { + expect(hubConditions(1, undefined)).toEqual([]); + expect(hubConditions(2, undefined)).toEqual([expect.objectContaining({ key: 'hub:ready' })]); + }); + + it('follows the edited rules', () => { + const rules = { + ...defaultRules, + 'hub-errors': { ...defaultRules['hub-errors'], threshold: 5 }, + 'hub-ready': { ...defaultRules['hub-ready'], enabled: false }, + }; + expect(hubConditions(9, metrics(0.04), rules)).toEqual([]); + }); +}); + +describe('hubRunning', () => { + it('looks for the running hub app container', () => { + const view = { project: 'graphite-hub', service: 'app', state: 'running' } as ContainerView; + expect(hubRunning([view])).toBe(true); + expect(hubRunning([{ ...view, state: 'exited' }])).toBe(false); + expect(hubRunning([{ ...view, project: 'strata' }])).toBe(false); + }); +}); + +describe('signedHeaders', () => { + it('signs an empty body the way the hub verifies it', () => { + const headers = signedHeaders('shared', 1_700_000_000_000); + expect(headers['X-Graphite-Timestamp']).toBe('1700000000'); + expect(headers['X-Graphite-Signature']).toBe(createHmac('sha256', 'shared').update('1700000000.').digest('hex')); + }); +});