feat: alert on the hub's error rate and failing readiness

app/src/server/alerts.ts

@@ -27,6 +27,8 @@ export const ruleKinds = {
   'container-unhealthy': { label: 'A container unhealthy', unit: null, threshold: 0, minutes: 1 },
   'container-restarts': { label: 'A container restarted in 10 minutes, times', unit: 'restarts', threshold: 3, minutes: 0 },
   'backup-late': { label: 'A project without a backup for', unit: 'hours', threshold: 26, minutes: 0 },
+  'hub-errors': { label: 'Hub requests failing over five minutes above', unit: '%', threshold: 1, minutes: 0 },
+  'hub-ready': { label: 'Hub readiness failing, checks in a row', unit: 'checks', threshold: 2, minutes: 0 },
 } as const;
 
 export type RuleKind = keyof typeof ruleKinds;

app/src/server/config.ts

@@ -33,6 +33,7 @@ export const config = {
   self: process.env.HOSTNAME ?? '',
   sshTarget: process.env.SSH_TARGET ?? 'ubuntu@localhost',
   secretsKey: process.env.SECRETS_KEY || undefined,
+  deployReportSecret: process.env.DEPLOY_REPORT_SECRET || undefined,
   graphiteHome: process.env.GRAPHITE_HOME ?? '/srv/graphite',
   platformRepo: process.env.PLATFORM_REPO ?? 'project-graphite/platform',
   platformApp:

app/src/server/hub.ts

@@ -0,0 +1,72 @@
+import { createHmac } from 'node:crypto';
+import { defaultRules, type Condition, type Rule, type RuleKind } from './alerts.ts';
+import { config } from './config.ts';
+import type { ContainerView } from './inventory.ts';
+
+export interface HubMetrics {
+  windowMinutes: number;
+  requests: number;
+  errorRate: number;
+  p95Ms: number | null;
+  liveConnections: number;
+}
+
+type Rules = Record<RuleKind, Rule>;
+
+const HUB = 'http://graphite-hub:3000';
+
+let failedChecks = 0;
+
+export function hubRunning(views: ContainerView[]) {
+  return views.some((container) => container.project === 'graphite-hub' && container.service === 'app' && container.state === 'running');
+}
+
+export function hubConditions(failures: number, metrics: HubMetrics | undefined, rules: Rules = defaultRules): Condition[] {
+  const percent = metrics ? 100 * metrics.errorRate : 0;
+  return [
+    ...(rules['hub-ready'].enabled && failures >= rules['hub-ready'].threshold
+      ? [{ key: 'hub:ready', title: 'The hub is not ready', detail: `/health/ready failed ${failures} checks in a row.` }]
+      : []),
+    ...(metrics && rules['hub-errors'].enabled && percent > rules['hub-errors'].threshold
+      ? [
+          {
+            key: 'hub:errors',
+            title: `The hub answers ${percent.toFixed(1)}% of requests with errors`,
+            detail: `Over ${rules['hub-errors'].threshold}% of ${metrics.requests} requests in the last ${metrics.windowMinutes} minutes; p95 ${metrics.p95Ms ?? 0} ms.`,
+            for: rules['hub-errors'].minutes * 60_000,
+          },
+        ]
+      : []),
+  ];
+}
+
+export function signedHeaders(secret: string, now = Date.now()) {
+  const stamp = String(Math.floor(now / 1000));
+  return { 'X-Graphite-Timestamp': stamp, 'X-Graphite-Signature': createHmac('sha256', secret).update(`${stamp}.`).digest('hex') };
+}
+
+export async function checkHub(rules: Rules) {
+  const secret = config.deployReportSecret;
+  if (!secret) {
+    return [];
+  }
+  const ready = await fetch(`${HUB}/health/ready`, { signal: AbortSignal.timeout(5_000) }).then(
+    (response) => response.ok,
+    () => false,
+  );
+  failedChecks = ready ? 0 : failedChecks + 1;
+  if (!ready) {
+    return hubConditions(failedChecks, undefined, rules);
+  }
+  const response = await fetch(`${HUB}/internal/metrics`, { headers: signedHeaders(secret), signal: AbortSignal.timeout(5_000) });
+  if (!response.ok) {
+    return [
+      {
+        key: 'hub:metrics',
+        title: 'The console cannot read the hub metrics',
+        detail: `The hub answered ${response.status}; DEPLOY_REPORT_SECRET must match in the console and hub env files.`,
+      },
+    ];
+  }
+  return hubConditions(failedChecks, (await response.json()) as HubMetrics, rules);
+}

app/src/server/monitor.ts

@@ -3,6 +3,7 @@ import { defaultRules, type Alert, type Alerts, type Condition, type Rule, type
 import { backupConditions } from './backups.ts';
 import { config } from './config.ts';
 import { docker } from './docker.ts';
+import { checkHub, hubRunning } from './hub.ts';
 import { snapshot, type ContainerView } from './inventory.ts';
 import { latest, type Frame, type HostLive } from './live.ts';
 import { Refusal } from './refusal.ts';
@@ -109,6 +110,7 @@ async function conditions(rules: Rules, now: number) {
     ...containerConditions(views, inspects, now, rules),
     ...usageConditions(frame, rules),
     ...(rules['backup-late'].enabled ? await backupConditions(now, undefined, rules['backup-late'].threshold) : []),
+    ...(hubRunning(views) ? await checkHub(rules) : []),
   ];
 }
 

app/test/hub.spec.ts

@@ -0,0 +1,47 @@
+import { createHmac } from 'node:crypto';
+import { describe, expect, it } from 'vitest';
+import { defaultRules } from '../src/server/alerts.ts';
+import { hubConditions, hubRunning, signedHeaders, type HubMetrics } from '../src/server/hub.ts';
+import type { ContainerView } from '../src/server/inventory.ts';
+
+const metrics = (errorRate: number): HubMetrics => ({ windowMinutes: 5, requests: 400, errorRate, p95Ms: 42, liveConnections: 3 });
+
+describe('hubConditions', () => {
+  it('alerts when more than one percent of requests fail over the window', () => {
+    expect(hubConditions(0, metrics(0.01))).toEqual([]);
+    expect(hubConditions(0, metrics(0.025))).toEqual([
+      expect.objectContaining({ key: 'hub:errors', title: 'The hub answers 2.5% of requests with errors' }),
+    ]);
+  });
+
+  it('alerts when readiness fails twice in a row', () => {
+    expect(hubConditions(1, undefined)).toEqual([]);
+    expect(hubConditions(2, undefined)).toEqual([expect.objectContaining({ key: 'hub:ready' })]);
+  });
+
+  it('follows the edited rules', () => {
+    const rules = {
+      ...defaultRules,
+      'hub-errors': { ...defaultRules['hub-errors'], threshold: 5 },
+      'hub-ready': { ...defaultRules['hub-ready'], enabled: false },
+    };
+    expect(hubConditions(9, metrics(0.04), rules)).toEqual([]);
+  });
+});
+
+describe('hubRunning', () => {
+  it('looks for the running hub app container', () => {
+    const view = { project: 'graphite-hub', service: 'app', state: 'running' } as ContainerView;
+    expect(hubRunning([view])).toBe(true);
+    expect(hubRunning([{ ...view, state: 'exited' }])).toBe(false);
+    expect(hubRunning([{ ...view, project: 'strata' }])).toBe(false);
+  });
+});
+
+describe('signedHeaders', () => {
+  it('signs an empty body the way the hub verifies it', () => {
+    const headers = signedHeaders('shared', 1_700_000_000_000);
+    expect(headers['X-Graphite-Timestamp']).toBe('1700000000');
+    expect(headers['X-Graphite-Signature']).toBe(createHmac('sha256', 'shared').update('1700000000.').digest('hex'));
+  });
+});