feat: live monitoring, minute averages and peaks, editable alert rules with email, log search

app/package.json

@@ -18,13 +18,15 @@
     "@xterm/addon-fit": "0.11.0",
     "@xterm/xterm": "6.0.0",
     "marked": "18.0.14",
+    "nodemailer": "10.3.1",
     "react": "19.3.0",
     "react-dom": "19.3.0",
     "ws": "8.22.0"
   },
   "devDependencies": {
     "@tailwindcss/vite": "4.3.3",
     "@types/node": "26.6.5",
+    "@types/nodemailer": "8.0.2",
     "@types/react": "19.3.0",
     "@types/react-dom": "19.3.0",
     "@types/ws": "8.18.2",

app/src/server/alerts.ts

@@ -6,22 +6,49 @@ export interface Condition {
   key: string;
   title: string;
   detail: string;
+  for?: number;
 }
 
-export interface Alert extends Condition {
+export interface Alert extends Omit<Condition, 'for'> {
   id: number;
   since: number;
   resolved: number | null;
   issue: number | null;
 }
 
-const confirmAfter = 2;
+export const ruleKinds = {
+  'host-cpu': { label: 'Host CPU above', unit: '%', threshold: 90, minutes: 5 },
+  'host-memory': { label: 'Host memory above', unit: '%', threshold: 90, minutes: 2 },
+  'host-swap': { label: 'Swap above', unit: '%', threshold: 80, minutes: 2 },
+  'host-disk': { label: 'Disk above', unit: '%', threshold: 85, minutes: 2 },
+  'container-cpu': { label: 'A container above its CPU limit by', unit: '%', threshold: 90, minutes: 5 },
+  'container-memory': { label: 'A container above its memory limit by', unit: '%', threshold: 90, minutes: 2 },
+  'container-down': { label: 'A container stopped', unit: null, threshold: 0, minutes: 1 },
+  'container-unhealthy': { label: 'A container unhealthy', unit: null, threshold: 0, minutes: 1 },
+  'container-restarts': { label: 'A container restarted in 10 minutes, times', unit: 'restarts', threshold: 3, minutes: 0 },
+  'backup-late': { label: 'A project without a backup for', unit: 'hours', threshold: 26, minutes: 0 },
+} as const;
+
+export type RuleKind = keyof typeof ruleKinds;
+
+export interface Rule {
+  kind: RuleKind;
+  threshold: number;
+  minutes: number;
+  enabled: boolean;
+}
+
+export const defaultRules = Object.fromEntries(
+  Object.entries(ruleKinds).map(([kind, rule]) => [kind, { kind, threshold: rule.threshold, minutes: rule.minutes, enabled: true }]),
+) as Record<RuleKind, Rule>;
+
+const resolveAfter = 60_000;
 const keepDays = 30;
 
 export class Alerts {
   private readonly db: DatabaseSync;
-  private readonly present = new Map<string, number>();
-  private readonly absent = new Map<string, number>();
+  private readonly since = new Map<string, number>();
+  private readonly clear = new Map<string, number>();
 
   constructor(directory: string) {
     mkdirSync(directory, { recursive: true });
@@ -32,9 +59,25 @@ export class Alerts {
         since INTEGER NOT NULL, resolved INTEGER, issue INTEGER
       );
       CREATE INDEX IF NOT EXISTS alerts_open ON alerts (key) WHERE resolved IS NULL;
+      CREATE TABLE IF NOT EXISTS rules (kind TEXT PRIMARY KEY, threshold REAL NOT NULL, minutes REAL NOT NULL, enabled INTEGER NOT NULL);
     `);
   }
 
+  rules(): Record<RuleKind, Rule> {
+    const saved = this.db.prepare('SELECT kind, threshold, minutes, enabled FROM rules').all() as { kind: string; threshold: number; minutes: number; enabled: number }[];
+    const rules = structuredClone(defaultRules);
+    for (const row of saved.filter((entry) => entry.kind in ruleKinds)) {
+      rules[row.kind as RuleKind] = { kind: row.kind as RuleKind, threshold: row.threshold, minutes: row.minutes, enabled: row.enabled === 1 };
+    }
+    return rules;
+  }
+
+  setRule(rule: Rule) {
+    this.db
+      .prepare('INSERT INTO rules (kind, threshold, minutes, enabled) VALUES (?, ?, ?, ?) ON CONFLICT (kind) DO UPDATE SET threshold = excluded.threshold, minutes = excluded.minutes, enabled = excluded.enabled')
+      .run(rule.kind, rule.threshold, rule.minutes, rule.enabled ? 1 : 0);
+  }
+
   active() {
     return this.db.prepare('SELECT * FROM alerts WHERE resolved IS NULL ORDER BY since').all() as unknown as Alert[];
   }
@@ -51,35 +94,34 @@ export class Alerts {
     const opened: Alert[] = [];
     const resolved: Alert[] = [];
     for (const condition of conditions) {
-      this.absent.delete(condition.key);
-      const streak = (this.present.get(condition.key) ?? 0) + 1;
-      this.present.set(condition.key, streak);
+      this.clear.delete(condition.key);
+      const first = this.since.get(condition.key) ?? now;
+      this.since.set(condition.key, first);
       const alert = open.get(condition.key);
       if (alert) {
         this.db.prepare('UPDATE alerts SET title = ?, detail = ? WHERE id = ?').run(condition.title, condition.detail, alert.id);
-      } else if (streak >= confirmAfter) {
+      } else if (now - first >= (condition.for ?? 0)) {
         const { lastInsertRowid } = this.db
           .prepare('INSERT INTO alerts (key, title, detail, since) VALUES (?, ?, ?, ?)')
-          .run(condition.key, condition.title, condition.detail, now);
-        opened.push({ ...condition, id: Number(lastInsertRowid), since: now, resolved: null, issue: null });
+          .run(condition.key, condition.title, condition.detail, first);
+        opened.push({ key: condition.key, title: condition.title, detail: condition.detail, id: Number(lastInsertRowid), since: first, resolved: null, issue: null });
       }
     }
     for (const alert of open.values()) {
       if (seen.has(alert.key)) {
         continue;
       }
-      this.present.delete(alert.key);
-      const streak = (this.absent.get(alert.key) ?? 0) + 1;
-      this.absent.set(alert.key, streak);
-      if (streak >= confirmAfter) {
+      const cleared = this.clear.get(alert.key) ?? now;
+      this.clear.set(alert.key, cleared);
+      if (now - cleared >= resolveAfter) {
         this.db.prepare('UPDATE alerts SET resolved = ? WHERE id = ?').run(now, alert.id);
-        this.absent.delete(alert.key);
+        this.clear.delete(alert.key);
         resolved.push({ ...alert, resolved: now });
       }
     }
-    for (const key of this.present.keys()) {
+    for (const key of this.since.keys()) {
       if (!seen.has(key)) {
-        this.present.delete(key);
+        this.since.delete(key);
       }
     }
     this.db.prepare('DELETE FROM alerts WHERE resolved IS NOT NULL AND resolved < ?').run(now - keepDays * 86400_000);

app/src/server/backups.ts

@@ -79,13 +79,13 @@ export function stampTime(stamp: string) {
   return Date.parse(`${stamp.slice(0, 4)}-${stamp.slice(4, 6)}-${stamp.slice(6, 8)}T${stamp.slice(9, 11)}:${stamp.slice(11, 13)}:${stamp.slice(13, 15)}Z`);
 }
 
-export async function backupConditions(now: number, root = backupRoot()) {
+export async function backupConditions(now: number, root = backupRoot(), hours = 26) {
   return (await listBackups(root)).flatMap(({ project, archives }) => {
     const latest = archives.find((archive) => !archive.imported);
     if (!latest) {
       return [];
     }
-    if (now - stampTime(latest.stamp) > 26 * hour) {
+    if (now - stampTime(latest.stamp) > hours * hour) {
       return [{ key: `backup:${project}`, title: `${project} has no recent backup`, detail: `The newest backup is ${Math.round((now - stampTime(latest.stamp)) / hour)} hours old.` }];
     }
     if (!latest.ok) {

app/src/server/config.ts

@@ -20,6 +20,9 @@ export const config = {
   organisation: process.env.GITHUB_ORG ?? 'Project-Graphite',
   alertsToken: process.env.ALERTS_TOKEN || undefined,
   alertsRepo: process.env.ALERTS_REPO ?? 'project-graphite/platform',
+  smtpUrl: process.env.SMTP_URL || undefined,
+  alertsEmail: process.env.ALERTS_EMAIL || undefined,
+  alertsFrom: process.env.ALERTS_FROM || undefined,
   docsRepo: process.env.DOCS_REPO ?? 'project-graphite/docs',
   dockerSocket: process.env.DOCKER_SOCKET ?? '/var/run/docker.sock',
   hostRoot: process.env.HOST_ROOT ?? '/proc/1/root',

app/src/server/docker.ts

@@ -16,9 +16,26 @@ export interface DockerContainer {
 }
 
 export interface DockerStats {
+  read?: string;
   cpu_stats: { cpu_usage: { total_usage: number }; system_cpu_usage?: number; online_cpus?: number };
   precpu_stats: { cpu_usage: { total_usage: number }; system_cpu_usage?: number };
   memory_stats: { usage?: number; limit?: number; stats?: { inactive_file?: number } };
+  networks?: Record<string, { rx_bytes: number; tx_bytes: number }>;
+  blkio_stats?: { io_service_bytes_recursive?: { op: string; value: number }[] | null };
+}
+
+export function networkTotals(stats: DockerStats) {
+  return Object.values(stats.networks ?? {}).reduce((sum, network) => ({ rx: sum.rx + network.rx_bytes, tx: sum.tx + network.tx_bytes }), { rx: 0, tx: 0 });
+}
+
+export function ioTotals(stats: DockerStats) {
+  return (stats.blkio_stats?.io_service_bytes_recursive ?? []).reduce(
+    (sum, entry) => ({
+      read: sum.read + (entry.op.toLowerCase() === 'read' ? entry.value : 0),
+      write: sum.write + (entry.op.toLowerCase() === 'write' ? entry.value : 0),
+    }),
+    { read: 0, write: 0 },
+  );
 }
 
 function call(method: string, path: string, body?: unknown, headers: Record<string, string> = {}) {

app/src/server/history.ts

@@ -21,13 +21,21 @@ export const keep = ranges['30d'].span;
 export interface Sample {
   target: string;
   cpu: number;
+  cpuPeak: number;
   memory: number;
+  memoryPeak: number;
+  net: number;
+  io: number;
 }
 
 export interface Point {
   at: number;
   cpu: number;
+  cpuPeak: number;
   memory: number;
+  memoryPeak: number;
+  net: number;
+  io: number;
 }
 
 export function projectTotals(containers: { name: string; project: string }[], seriesOf: (name: string) => Point[], cpus: number) {
@@ -36,9 +44,13 @@ export function projectTotals(containers: { name: string; project: string }[], s
     const totals = projects.get(container.project) ?? new Map<number, Point>();
     projects.set(container.project, totals);
     for (const point of seriesOf(container.name)) {
-      const total = totals.get(point.at) ?? { at: point.at, cpu: 0, memory: 0 };
+      const total = totals.get(point.at) ?? { at: point.at, cpu: 0, cpuPeak: 0, memory: 0, memoryPeak: 0, net: 0, io: 0 };
       total.cpu += point.cpu / cpus;
+      total.cpuPeak += point.cpuPeak / cpus;
       total.memory += point.memory;
+      total.memoryPeak += point.memoryPeak;
+      total.net += point.net;
+      total.io += point.io;
       totals.set(point.at, total);
     }
   }
@@ -58,12 +70,18 @@ export class History {
       CREATE INDEX IF NOT EXISTS samples_target_at ON samples (target, at);
       CREATE INDEX IF NOT EXISTS samples_at ON samples (at);
     `);
+    const columns = new Set((this.db.prepare('PRAGMA table_info(samples)').all() as { name: string }[]).map((column) => column.name));
+    for (const [name, type] of [['cpu_peak', 'REAL'], ['memory_peak', 'INTEGER'], ['net', 'REAL'], ['io', 'REAL']]) {
+      if (!columns.has(name)) {
+        this.db.exec(`ALTER TABLE samples ADD COLUMN ${name} ${type}`);
+      }
+    }
   }
 
   record(at: number, samples: Sample[]) {
-    const insert = this.db.prepare('INSERT INTO samples (at, target, cpu, memory) VALUES (?, ?, ?, ?)');
+    const insert = this.db.prepare('INSERT INTO samples (at, target, cpu, cpu_peak, memory, memory_peak, net, io) VALUES (?, ?, ?, ?, ?, ?, ?, ?)');
     for (const sample of samples) {
-      insert.run(at, sample.target, sample.cpu, sample.memory);
+      insert.run(at, sample.target, sample.cpu, sample.cpuPeak, sample.memory, sample.memoryPeak, sample.net, sample.io);
     }
     this.db.prepare('DELETE FROM samples WHERE at < ?').run(at - keep);
   }
@@ -72,7 +90,9 @@ export class History {
     const { span, step } = ranges[range];
     return this.db
       .prepare(
-        `SELECT CAST(at / ? AS INTEGER) * ? AS at, avg(cpu) AS cpu, CAST(avg(memory) AS INTEGER) AS memory
+        `SELECT CAST(at / ? AS INTEGER) * ? AS at, avg(cpu) AS cpu, max(coalesce(cpu_peak, cpu)) AS cpuPeak,
+           CAST(avg(memory) AS INTEGER) AS memory, max(coalesce(memory_peak, memory)) AS memoryPeak,
+           coalesce(avg(net), 0) AS net, coalesce(avg(io), 0) AS io
          FROM samples WHERE target = ? AND at > ? GROUP BY 1 ORDER BY 1`,
       )
       .all(step, step, target, now - span) as unknown as Point[];

app/src/server/host.ts

@@ -2,8 +2,6 @@ import { readFile, statfs } from 'node:fs/promises';
 import os from 'node:os';
 import { config } from './config.ts';
 
-let previous: { idle: number; total: number } | undefined;
-
 export function cpuTimes(stat: string) {
   const fields = stat.split('\n')[0].trim().split(/\s+/).slice(1).map(Number);
   const idle = fields[3] + (fields[4] ?? 0);
@@ -23,23 +21,39 @@ export function meminfo(text: string) {
   };
 }
 
-export async function hostStats() {
-  const [stat, memory, load, uptime, disk] = await Promise.all([
-    readFile('/proc/stat', 'utf8'),
-    readFile('/proc/meminfo', 'utf8'),
-    readFile('/proc/loadavg', 'utf8'),
-    readFile('/proc/uptime', 'utf8'),
-    statfs(config.hostRoot).catch(() => statfs('/')),
-  ]);
-  const times = cpuTimes(stat);
-  const cpu = previous ? 100 * (1 - (times.idle - previous.idle) / Math.max(1, times.total - previous.total)) : 0;
-  previous = times;
+export function networkBytes(text: string) {
+  return text
+    .split('\n')
+    .slice(2)
+    .map((line) => line.trim().split(/[:\s]+/))
+    .filter((fields) => fields.length > 9 && !/^(lo|veth|br-|docker)/.test(fields[0]))
+    .reduce((sum, fields) => ({ rx: sum.rx + Number(fields[1]), tx: sum.tx + Number(fields[9]) }), { rx: 0, tx: 0 });
+}
+
+export function diskBytes(text: string) {
+  return text
+    .split('\n')
+    .map((line) => line.trim().split(/\s+/))
+    .filter((fields) => /^(sd[a-z]+|vd[a-z]+|xvd[a-z]+|nvme\d+n\d+)$/.test(fields[2] ?? ''))
+    .reduce((sum, fields) => ({ read: sum.read + Number(fields[5]) * 512, write: sum.write + Number(fields[9]) * 512 }), { read: 0, write: 0 });
+}
+
+export async function readHost() {
+  const [stat, memory, load, uptime, network, disks] = await Promise.all(
+    ['/proc/stat', '/proc/meminfo', '/proc/loadavg', '/proc/uptime', '/proc/1/net/dev', '/proc/diskstats'].map((file) => readFile(file, 'utf8')),
+  );
   return {
-    cpu: Math.max(0, cpu),
+    times: cpuTimes(stat),
     cpus: os.cpus().length,
     load: load.split(' ').slice(0, 3).map(Number),
     uptime: Number(uptime.split(' ')[0]),
-    disk: { used: (disk.blocks - disk.bfree) * disk.bsize, total: (disk.blocks - disk.bfree + disk.bavail) * disk.bsize },
     ...meminfo(memory),
+    network: networkBytes(network),
+    io: diskBytes(disks),
   };
 }
+
+export async function diskSpace() {
+  const disk = await statfs(config.hostRoot).catch(() => statfs('/'));
+  return { used: (disk.blocks - disk.bfree) * disk.bsize, total: (disk.blocks - disk.bfree + disk.bavail) * disk.bsize };
+}