From 29d1bb6a0d5eda57f55c498379361872a657cc38 Mon Sep 17 00:00:00 2001 From: Amr Mohamed El-Sheraey <141947355+AmrMsCLL@users.noreply.github.com> Date: Thu, 5 Nov 2026 13:06:06 +0200 Subject: [PATCH] feat: live monitoring every second, minute averages and peaks, editable alert rules with email, and log search (#21) A stats stream per running container and /proc on the host feed a 1 second frame: CPU, memory, network and disk rates. GET /api/live streams it to every tab over Server-Sent Events with the last three minutes, and closes when the viewer stops being a member. History keeps each minute's average and peak, with network and disk, in the existing table. Charts gain a live range, network and disk, and a peak view. Alerts are checked every 10 seconds against rules owners edit (threshold, duration, on or off), open once a condition holds for its duration, resolve after a clear minute, and are emailed through SMTP_URL to ALERTS_EMAIL with a test button. Logs adds a search across containers. --- app/package-lock.json | 21 +++ app/package.json | 2 + app/src/server/alerts.ts | 76 +++++++-- app/src/server/backups.ts | 4 +- app/src/server/config.ts | 3 + app/src/server/docker.ts | 17 ++ app/src/server/history.ts | 28 +++- app/src/server/host.ts | 44 +++-- app/src/server/inventory.ts | 67 +++----- app/src/server/live.ts | 244 ++++++++++++++++++++++++++++ app/src/server/main.ts | 49 +++++- app/src/server/monitor.ts | 127 ++++++++++++--- app/src/server/terminal.ts | 23 ++- app/src/web/App.tsx | 7 +- app/src/web/api.ts | 49 ++++++ app/src/web/components/Chart.tsx | 7 +- app/src/web/components/Metrics.tsx | 57 +++++-- app/src/web/live.ts | 65 ++++++++ app/src/web/pages/Alerts.tsx | 118 +++++++++++++- app/src/web/pages/ContainerPage.tsx | 2 +- app/src/web/pages/LogSearch.tsx | 73 +++++++++ app/src/web/pages/Overview.tsx | 46 ++++-- app/src/web/pages/ProjectPage.tsx | 2 +- app/test/alerts.spec.ts | 73 +++++++-- app/test/history.spec.ts | 72 +++++--- app/test/host.spec.ts | 23 ++- app/test/live.spec.ts | 42 +++++ 27 files changed, 1147 insertions(+), 194 deletions(-) create mode 100644 app/src/server/live.ts create mode 100644 app/src/web/live.ts create mode 100644 app/src/web/pages/LogSearch.tsx create mode 100644 app/test/live.spec.ts diff --git a/app/package-lock.json b/app/package-lock.json index 2972a15..be1ce05 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -15,6 +15,7 @@ "@xterm/addon-fit": "0.11.0", "@xterm/xterm": "6.0.0", "marked": "18.0.14", + "nodemailer": "10.3.1", "react": "19.3.0", "react-dom": "19.3.0", "ws": "8.22.0" @@ -22,6 +23,7 @@ "devDependencies": { "@tailwindcss/vite": "4.3.3", "@types/node": "26.6.5", + "@types/nodemailer": "8.0.2", "@types/react": "19.3.0", "@types/react-dom": "19.3.0", "@types/ws": "8.18.2", @@ -1497,6 +1499,16 @@ "undici-types": "~8.9.0" } }, + "node_modules/@types/nodemailer": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@types/nodemailer/-/nodemailer-8.0.2.tgz", + "integrity": "sha512-c7M5ox8p0nEOfbJ2E9Qcmt8/QCu/VzsiAhzBiZbvky2PhKZDHpKB3sQHBM5MEZNkCfBOaS6S9//AGPRTNB7IaA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/parse-json": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz", @@ -2317,6 +2329,15 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/nodemailer": { + "version": "10.3.1", + "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-10.3.1.tgz", + "integrity": "sha512-uF0AFsJC07Y1vdW7p580DKAnh7rAO+4UPSSB3mDCPNDufoiaOebV6JmrzpxB14OjEEEiBsNdYoM2Y+fN8M7BBA==", + "license": "MIT-0", + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", diff --git a/app/package.json b/app/package.json index 69a5c28..83d5263 100644 --- a/app/package.json +++ b/app/package.json @@ -18,6 +18,7 @@ "@xterm/addon-fit": "0.11.0", "@xterm/xterm": "6.0.0", "marked": "18.0.14", + "nodemailer": "10.3.1", "react": "19.3.0", "react-dom": "19.3.0", "ws": "8.22.0" @@ -25,6 +26,7 @@ "devDependencies": { "@tailwindcss/vite": "4.3.3", "@types/node": "26.6.5", + "@types/nodemailer": "8.0.2", "@types/react": "19.3.0", "@types/react-dom": "19.3.0", "@types/ws": "8.18.2", diff --git a/app/src/server/alerts.ts b/app/src/server/alerts.ts index 0969cf3..3e11927 100644 --- a/app/src/server/alerts.ts +++ b/app/src/server/alerts.ts @@ -6,22 +6,49 @@ export interface Condition { key: string; title: string; detail: string; + for?: number; } -export interface Alert extends Condition { +export interface Alert extends Omit { id: number; since: number; resolved: number | null; issue: number | null; } -const confirmAfter = 2; +export const ruleKinds = { + 'host-cpu': { label: 'Host CPU above', unit: '%', threshold: 90, minutes: 5 }, + 'host-memory': { label: 'Host memory above', unit: '%', threshold: 90, minutes: 2 }, + 'host-swap': { label: 'Swap above', unit: '%', threshold: 80, minutes: 2 }, + 'host-disk': { label: 'Disk above', unit: '%', threshold: 85, minutes: 2 }, + 'container-cpu': { label: 'A container above its CPU limit by', unit: '%', threshold: 90, minutes: 5 }, + 'container-memory': { label: 'A container above its memory limit by', unit: '%', threshold: 90, minutes: 2 }, + 'container-down': { label: 'A container stopped', unit: null, threshold: 0, minutes: 1 }, + 'container-unhealthy': { label: 'A container unhealthy', unit: null, threshold: 0, minutes: 1 }, + 'container-restarts': { label: 'A container restarted in 10 minutes, times', unit: 'restarts', threshold: 3, minutes: 0 }, + 'backup-late': { label: 'A project without a backup for', unit: 'hours', threshold: 26, minutes: 0 }, +} as const; + +export type RuleKind = keyof typeof ruleKinds; + +export interface Rule { + kind: RuleKind; + threshold: number; + minutes: number; + enabled: boolean; +} + +export const defaultRules = Object.fromEntries( + Object.entries(ruleKinds).map(([kind, rule]) => [kind, { kind, threshold: rule.threshold, minutes: rule.minutes, enabled: true }]), +) as Record; + +const resolveAfter = 60_000; const keepDays = 30; export class Alerts { private readonly db: DatabaseSync; - private readonly present = new Map(); - private readonly absent = new Map(); + private readonly since = new Map(); + private readonly clear = new Map(); constructor(directory: string) { mkdirSync(directory, { recursive: true }); @@ -32,9 +59,25 @@ export class Alerts { since INTEGER NOT NULL, resolved INTEGER, issue INTEGER ); CREATE INDEX IF NOT EXISTS alerts_open ON alerts (key) WHERE resolved IS NULL; + CREATE TABLE IF NOT EXISTS rules (kind TEXT PRIMARY KEY, threshold REAL NOT NULL, minutes REAL NOT NULL, enabled INTEGER NOT NULL); `); } + rules(): Record { + const saved = this.db.prepare('SELECT kind, threshold, minutes, enabled FROM rules').all() as { kind: string; threshold: number; minutes: number; enabled: number }[]; + const rules = structuredClone(defaultRules); + for (const row of saved.filter((entry) => entry.kind in ruleKinds)) { + rules[row.kind as RuleKind] = { kind: row.kind as RuleKind, threshold: row.threshold, minutes: row.minutes, enabled: row.enabled === 1 }; + } + return rules; + } + + setRule(rule: Rule) { + this.db + .prepare('INSERT INTO rules (kind, threshold, minutes, enabled) VALUES (?, ?, ?, ?) ON CONFLICT (kind) DO UPDATE SET threshold = excluded.threshold, minutes = excluded.minutes, enabled = excluded.enabled') + .run(rule.kind, rule.threshold, rule.minutes, rule.enabled ? 1 : 0); + } + active() { return this.db.prepare('SELECT * FROM alerts WHERE resolved IS NULL ORDER BY since').all() as unknown as Alert[]; } @@ -51,35 +94,34 @@ export class Alerts { const opened: Alert[] = []; const resolved: Alert[] = []; for (const condition of conditions) { - this.absent.delete(condition.key); - const streak = (this.present.get(condition.key) ?? 0) + 1; - this.present.set(condition.key, streak); + this.clear.delete(condition.key); + const first = this.since.get(condition.key) ?? now; + this.since.set(condition.key, first); const alert = open.get(condition.key); if (alert) { this.db.prepare('UPDATE alerts SET title = ?, detail = ? WHERE id = ?').run(condition.title, condition.detail, alert.id); - } else if (streak >= confirmAfter) { + } else if (now - first >= (condition.for ?? 0)) { const { lastInsertRowid } = this.db .prepare('INSERT INTO alerts (key, title, detail, since) VALUES (?, ?, ?, ?)') - .run(condition.key, condition.title, condition.detail, now); - opened.push({ ...condition, id: Number(lastInsertRowid), since: now, resolved: null, issue: null }); + .run(condition.key, condition.title, condition.detail, first); + opened.push({ key: condition.key, title: condition.title, detail: condition.detail, id: Number(lastInsertRowid), since: first, resolved: null, issue: null }); } } for (const alert of open.values()) { if (seen.has(alert.key)) { continue; } - this.present.delete(alert.key); - const streak = (this.absent.get(alert.key) ?? 0) + 1; - this.absent.set(alert.key, streak); - if (streak >= confirmAfter) { + const cleared = this.clear.get(alert.key) ?? now; + this.clear.set(alert.key, cleared); + if (now - cleared >= resolveAfter) { this.db.prepare('UPDATE alerts SET resolved = ? WHERE id = ?').run(now, alert.id); - this.absent.delete(alert.key); + this.clear.delete(alert.key); resolved.push({ ...alert, resolved: now }); } } - for (const key of this.present.keys()) { + for (const key of this.since.keys()) { if (!seen.has(key)) { - this.present.delete(key); + this.since.delete(key); } } this.db.prepare('DELETE FROM alerts WHERE resolved IS NOT NULL AND resolved < ?').run(now - keepDays * 86400_000); diff --git a/app/src/server/backups.ts b/app/src/server/backups.ts index a1f81b6..15fefe6 100644 --- a/app/src/server/backups.ts +++ b/app/src/server/backups.ts @@ -79,13 +79,13 @@ export function stampTime(stamp: string) { return Date.parse(`${stamp.slice(0, 4)}-${stamp.slice(4, 6)}-${stamp.slice(6, 8)}T${stamp.slice(9, 11)}:${stamp.slice(11, 13)}:${stamp.slice(13, 15)}Z`); } -export async function backupConditions(now: number, root = backupRoot()) { +export async function backupConditions(now: number, root = backupRoot(), hours = 26) { return (await listBackups(root)).flatMap(({ project, archives }) => { const latest = archives.find((archive) => !archive.imported); if (!latest) { return []; } - if (now - stampTime(latest.stamp) > 26 * hour) { + if (now - stampTime(latest.stamp) > hours * hour) { return [{ key: `backup:${project}`, title: `${project} has no recent backup`, detail: `The newest backup is ${Math.round((now - stampTime(latest.stamp)) / hour)} hours old.` }]; } if (!latest.ok) { diff --git a/app/src/server/config.ts b/app/src/server/config.ts index 855a955..8d22d33 100644 --- a/app/src/server/config.ts +++ b/app/src/server/config.ts @@ -20,6 +20,9 @@ export const config = { organisation: process.env.GITHUB_ORG ?? 'Project-Graphite', alertsToken: process.env.ALERTS_TOKEN || undefined, alertsRepo: process.env.ALERTS_REPO ?? 'project-graphite/platform', + smtpUrl: process.env.SMTP_URL || undefined, + alertsEmail: process.env.ALERTS_EMAIL || undefined, + alertsFrom: process.env.ALERTS_FROM || undefined, docsRepo: process.env.DOCS_REPO ?? 'project-graphite/docs', dockerSocket: process.env.DOCKER_SOCKET ?? '/var/run/docker.sock', hostRoot: process.env.HOST_ROOT ?? '/proc/1/root', diff --git a/app/src/server/docker.ts b/app/src/server/docker.ts index ff4c8cf..3b57ca8 100644 --- a/app/src/server/docker.ts +++ b/app/src/server/docker.ts @@ -16,9 +16,26 @@ export interface DockerContainer { } export interface DockerStats { + read?: string; cpu_stats: { cpu_usage: { total_usage: number }; system_cpu_usage?: number; online_cpus?: number }; precpu_stats: { cpu_usage: { total_usage: number }; system_cpu_usage?: number }; memory_stats: { usage?: number; limit?: number; stats?: { inactive_file?: number } }; + networks?: Record; + blkio_stats?: { io_service_bytes_recursive?: { op: string; value: number }[] | null }; +} + +export function networkTotals(stats: DockerStats) { + return Object.values(stats.networks ?? {}).reduce((sum, network) => ({ rx: sum.rx + network.rx_bytes, tx: sum.tx + network.tx_bytes }), { rx: 0, tx: 0 }); +} + +export function ioTotals(stats: DockerStats) { + return (stats.blkio_stats?.io_service_bytes_recursive ?? []).reduce( + (sum, entry) => ({ + read: sum.read + (entry.op.toLowerCase() === 'read' ? entry.value : 0), + write: sum.write + (entry.op.toLowerCase() === 'write' ? entry.value : 0), + }), + { read: 0, write: 0 }, + ); } function call(method: string, path: string, body?: unknown, headers: Record = {}) { diff --git a/app/src/server/history.ts b/app/src/server/history.ts index f0fd6b8..37db658 100644 --- a/app/src/server/history.ts +++ b/app/src/server/history.ts @@ -21,13 +21,21 @@ export const keep = ranges['30d'].span; export interface Sample { target: string; cpu: number; + cpuPeak: number; memory: number; + memoryPeak: number; + net: number; + io: number; } export interface Point { at: number; cpu: number; + cpuPeak: number; memory: number; + memoryPeak: number; + net: number; + io: number; } export function projectTotals(containers: { name: string; project: string }[], seriesOf: (name: string) => Point[], cpus: number) { @@ -36,9 +44,13 @@ export function projectTotals(containers: { name: string; project: string }[], s const totals = projects.get(container.project) ?? new Map(); projects.set(container.project, totals); for (const point of seriesOf(container.name)) { - const total = totals.get(point.at) ?? { at: point.at, cpu: 0, memory: 0 }; + const total = totals.get(point.at) ?? { at: point.at, cpu: 0, cpuPeak: 0, memory: 0, memoryPeak: 0, net: 0, io: 0 }; total.cpu += point.cpu / cpus; + total.cpuPeak += point.cpuPeak / cpus; total.memory += point.memory; + total.memoryPeak += point.memoryPeak; + total.net += point.net; + total.io += point.io; totals.set(point.at, total); } } @@ -58,12 +70,18 @@ export class History { CREATE INDEX IF NOT EXISTS samples_target_at ON samples (target, at); CREATE INDEX IF NOT EXISTS samples_at ON samples (at); `); + const columns = new Set((this.db.prepare('PRAGMA table_info(samples)').all() as { name: string }[]).map((column) => column.name)); + for (const [name, type] of [['cpu_peak', 'REAL'], ['memory_peak', 'INTEGER'], ['net', 'REAL'], ['io', 'REAL']]) { + if (!columns.has(name)) { + this.db.exec(`ALTER TABLE samples ADD COLUMN ${name} ${type}`); + } + } } record(at: number, samples: Sample[]) { - const insert = this.db.prepare('INSERT INTO samples (at, target, cpu, memory) VALUES (?, ?, ?, ?)'); + const insert = this.db.prepare('INSERT INTO samples (at, target, cpu, cpu_peak, memory, memory_peak, net, io) VALUES (?, ?, ?, ?, ?, ?, ?, ?)'); for (const sample of samples) { - insert.run(at, sample.target, sample.cpu, sample.memory); + insert.run(at, sample.target, sample.cpu, sample.cpuPeak, sample.memory, sample.memoryPeak, sample.net, sample.io); } this.db.prepare('DELETE FROM samples WHERE at < ?').run(at - keep); } @@ -72,7 +90,9 @@ export class History { const { span, step } = ranges[range]; return this.db .prepare( - `SELECT CAST(at / ? AS INTEGER) * ? AS at, avg(cpu) AS cpu, CAST(avg(memory) AS INTEGER) AS memory + `SELECT CAST(at / ? AS INTEGER) * ? AS at, avg(cpu) AS cpu, max(coalesce(cpu_peak, cpu)) AS cpuPeak, + CAST(avg(memory) AS INTEGER) AS memory, max(coalesce(memory_peak, memory)) AS memoryPeak, + coalesce(avg(net), 0) AS net, coalesce(avg(io), 0) AS io FROM samples WHERE target = ? AND at > ? GROUP BY 1 ORDER BY 1`, ) .all(step, step, target, now - span) as unknown as Point[]; diff --git a/app/src/server/host.ts b/app/src/server/host.ts index dc2e951..2aa3cf3 100644 --- a/app/src/server/host.ts +++ b/app/src/server/host.ts @@ -2,8 +2,6 @@ import { readFile, statfs } from 'node:fs/promises'; import os from 'node:os'; import { config } from './config.ts'; -let previous: { idle: number; total: number } | undefined; - export function cpuTimes(stat: string) { const fields = stat.split('\n')[0].trim().split(/\s+/).slice(1).map(Number); const idle = fields[3] + (fields[4] ?? 0); @@ -23,23 +21,39 @@ export function meminfo(text: string) { }; } -export async function hostStats() { - const [stat, memory, load, uptime, disk] = await Promise.all([ - readFile('/proc/stat', 'utf8'), - readFile('/proc/meminfo', 'utf8'), - readFile('/proc/loadavg', 'utf8'), - readFile('/proc/uptime', 'utf8'), - statfs(config.hostRoot).catch(() => statfs('/')), - ]); - const times = cpuTimes(stat); - const cpu = previous ? 100 * (1 - (times.idle - previous.idle) / Math.max(1, times.total - previous.total)) : 0; - previous = times; +export function networkBytes(text: string) { + return text + .split('\n') + .slice(2) + .map((line) => line.trim().split(/[:\s]+/)) + .filter((fields) => fields.length > 9 && !/^(lo|veth|br-|docker)/.test(fields[0])) + .reduce((sum, fields) => ({ rx: sum.rx + Number(fields[1]), tx: sum.tx + Number(fields[9]) }), { rx: 0, tx: 0 }); +} + +export function diskBytes(text: string) { + return text + .split('\n') + .map((line) => line.trim().split(/\s+/)) + .filter((fields) => /^(sd[a-z]+|vd[a-z]+|xvd[a-z]+|nvme\d+n\d+)$/.test(fields[2] ?? '')) + .reduce((sum, fields) => ({ read: sum.read + Number(fields[5]) * 512, write: sum.write + Number(fields[9]) * 512 }), { read: 0, write: 0 }); +} + +export async function readHost() { + const [stat, memory, load, uptime, network, disks] = await Promise.all( + ['/proc/stat', '/proc/meminfo', '/proc/loadavg', '/proc/uptime', '/proc/1/net/dev', '/proc/diskstats'].map((file) => readFile(file, 'utf8')), + ); return { - cpu: Math.max(0, cpu), + times: cpuTimes(stat), cpus: os.cpus().length, load: load.split(' ').slice(0, 3).map(Number), uptime: Number(uptime.split(' ')[0]), - disk: { used: (disk.blocks - disk.bfree) * disk.bsize, total: (disk.blocks - disk.bfree + disk.bavail) * disk.bsize }, ...meminfo(memory), + network: networkBytes(network), + io: diskBytes(disks), }; } + +export async function diskSpace() { + const disk = await statfs(config.hostRoot).catch(() => statfs('/')); + return { used: (disk.blocks - disk.bfree) * disk.bsize, total: (disk.blocks - disk.bfree + disk.bavail) * disk.bsize }; +} diff --git a/app/src/server/inventory.ts b/app/src/server/inventory.ts index feccb30..1bb55f3 100644 --- a/app/src/server/inventory.ts +++ b/app/src/server/inventory.ts @@ -1,6 +1,6 @@ -import { cpuPercent, docker, memoryUsed, type DockerContainer, type DockerStats } from './docker.ts'; +import { docker, type DockerContainer } from './docker.ts'; import type { History } from './history.ts'; -import { hostStats } from './host.ts'; +import { latest, start, track, type ContainerLive, type HostLive } from './live.ts'; export interface ContainerView { id: string; @@ -29,10 +29,8 @@ export interface Disk { volumes: { name: string; project: string | null; size: number; inUse: boolean }[]; } -type Snapshot = { host: Awaited>; views: ContainerView[] }; - -let latest: Snapshot | undefined; -let measuring: Promise | undefined; +let listed: ContainerView[] | undefined; +let measuring: Promise | undefined; let disk: Disk | undefined; let reading: Promise | undefined; @@ -40,7 +38,7 @@ function health(status: string) { return /\((healthy|unhealthy|health: starting)\)/.exec(status)?.[1] ?? null; } -function view(container: DockerContainer, stats?: DockerStats): ContainerView { +function view(container: DockerContainer): ContainerView { return { id: container.Id, name: container.Names[0].replace(/^\//, ''), @@ -55,33 +53,30 @@ function view(container: DockerContainer, stats?: DockerStats): ContainerView { ports: [...new Set(container.Ports.map((port) => port.PrivatePort))].sort((a, b) => a - b), networks: Object.fromEntries(Object.entries(container.NetworkSettings.Networks).map(([name, network]) => [name, network.IPAddress])), sleeps: container.Labels['sablier.enable'] === 'true', - cpu: stats ? cpuPercent(stats) : null, - memory: stats ? memoryUsed(stats) : null, - memoryLimit: stats?.memory_stats.limit ?? null, + cpu: null, + memory: null, + memoryLimit: null, }; } -async function measure() { - const containers = await docker('GET', '/containers/json?all=1'); - const stats = await Promise.all( - containers.map((container) => - container.State === 'running' - ? docker('GET', `/containers/${container.Id}/stats?stream=false`).catch(() => undefined) - : undefined, - ), - ); - return containers.map((container, index) => view(container, stats[index])); +export function withLive(view: ContainerView, live: ContainerLive | undefined): ContainerView { + return view.state === 'running' && live ? { ...view, cpu: live.cpu, memory: live.memory, memoryLimit: live.memoryLimit } : view; } export function refresh() { - measuring ??= Promise.all([hostStats(), measure()]) - .then(([host, views]) => (latest = { host, views })) + measuring ??= docker('GET', '/containers/json?all=1') + .then((containers) => { + listed = containers.map(view); + track(listed); + return listed; + }) .finally(() => (measuring = undefined)); return measuring; } -export async function snapshot() { - return latest ?? (await refresh()); +export async function snapshot(): Promise<{ host: HostLive; views: ContainerView[] }> { + const [views, frame] = await Promise.all([listed ?? refresh(), latest()]); + return { host: frame.host, views: views.map((entry) => withLive(entry, frame.containers[entry.name])) }; } export async function containers() { @@ -151,25 +146,17 @@ export function projects(views: ContainerView[], usage?: Disk) { }); } -export function collect(history: History, { measureEvery = 5000, recordEvery = 60_000, diskEvery = 300_000 } = {}) { - const record = async () => { - const { host, views } = await snapshot(); - history.record(Date.now(), [ - { target: 'host', cpu: host.cpu, memory: host.memory.used }, - ...views.filter((container) => container.cpu !== null).map((container) => ({ - target: container.name, - cpu: container.cpu!, - memory: container.memory!, - })), - ]); - }; +export function collect(history: History, { listEvery = 5000, diskEvery = 300_000 } = {}) { const report = (task: string) => (error: unknown) => console.error(`${task} failed:`, error); + const stop = start(history); const timers = [ - setInterval(() => void refresh().catch(report('measure')), measureEvery), - setInterval(() => void record().catch(report('sample')), recordEvery), + setInterval(() => void refresh().catch(report('container list')), listEvery), setInterval(() => void refreshDisk().catch(report('disk usage')), diskEvery), ]; - void refresh().catch(report('measure')); + void refresh().catch(report('container list')); void refreshDisk().catch(report('disk usage')); - return () => timers.forEach(clearInterval); + return () => { + stop(); + timers.forEach(clearInterval); + }; } diff --git a/app/src/server/live.ts b/app/src/server/live.ts new file mode 100644 index 0000000..3938ae6 --- /dev/null +++ b/app/src/server/live.ts @@ -0,0 +1,244 @@ +import type http from 'node:http'; +import { cpuPercent, docker, ioTotals, memoryUsed, networkTotals, stream, type DockerStats } from './docker.ts'; +import type { History } from './history.ts'; +import { diskSpace, readHost } from './host.ts'; + +export interface Traffic { + rx: number; + tx: number; + read: number; + write: number; +} + +export interface ContainerLive extends Traffic { + project: string; + cpu: number; + cpuLimit: number | null; + memory: number; + memoryLimit: number | null; +} + +export interface HostLive extends Traffic { + cpu: number; + cpus: number; + load: number[]; + uptime: number; + memory: { used: number; total: number }; + swap: { used: number; total: number }; + disk: { used: number; total: number }; +} + +export interface Frame { + at: number; + host: HostLive; + containers: Record; +} + +interface Tracked { + name: string; + project: string; + cpuLimit: number | null; + response?: http.IncomingMessage; + last?: Traffic & { at: number }; + value?: ContainerLive; +} + +interface Sums { + seconds: number; + cpu: number; + cpuPeak: number; + memory: number; + memoryPeak: number; + net: number; + io: number; +} + +export const windowSeconds = 180; +const tracked = new Map(); +const frames: Frame[] = []; +const listeners = new Set<(frame: Frame) => void>(); +const sums = new Map(); +let minute = 0; +let previousHost: { at: number; idle: number; total: number; traffic: Traffic } | undefined; +let space = { used: 0, total: 0 }; +let ready: (frame: Frame) => void; +const first = new Promise((resolve) => (ready = resolve)); + +export function rate(now: number, before: number | undefined, seconds: number) { + return before === undefined || seconds <= 0 ? 0 : Math.max(0, (now - before) / seconds); +} + +function read(entry: Tracked, stats: DockerStats) { + const at = Date.parse(stats.read ?? '') || Date.now(); + const traffic = { ...networkTotals(stats), ...ioTotals(stats) }; + const seconds = entry.last ? (at - entry.last.at) / 1000 : 0; + entry.value = { + project: entry.project, + cpu: cpuPercent(stats), + cpuLimit: entry.cpuLimit, + memory: memoryUsed(stats), + memoryLimit: stats.memory_stats.limit ?? null, + rx: rate(traffic.rx, entry.last?.rx, seconds), + tx: rate(traffic.tx, entry.last?.tx, seconds), + read: rate(traffic.read, entry.last?.read, seconds), + write: rate(traffic.write, entry.last?.write, seconds), + }; + entry.last = { at, ...traffic }; +} + +async function follow(id: string, name: string, project: string) { + const entry: Tracked = { name, project, cpuLimit: null }; + tracked.set(id, entry); + try { + const { HostConfig } = await docker<{ HostConfig: { NanoCpus: number } }>('GET', `/containers/${id}/json`); + entry.cpuLimit = HostConfig.NanoCpus ? HostConfig.NanoCpus / 1e9 : null; + const response = await stream('GET', `/containers/${id}/stats?stream=1`); + if (tracked.get(id) !== entry) { + response.destroy(); + return; + } + entry.response = response; + let buffer = ''; + response.setEncoding('utf8'); + response.on('data', (chunk: string) => { + buffer += chunk; + for (let end = buffer.indexOf('\n'); end >= 0; end = buffer.indexOf('\n')) { + const line = buffer.slice(0, end).trim(); + buffer = buffer.slice(end + 1); + if (line) { + read(entry, JSON.parse(line) as DockerStats); + } + } + }); + response.on('close', () => { + if (tracked.get(id) === entry) { + tracked.delete(id); + } + }); + } catch (error) { + tracked.delete(id); + console.error(`following ${name} failed:`, error); + } +} + +export function track(views: { id: string; name: string; project: string; state: string }[]) { + const running = new Map(views.filter((view) => view.state === 'running').map((view) => [view.id, view])); + for (const [id, entry] of tracked) { + if (!running.has(id)) { + entry.response?.destroy(); + tracked.delete(id); + } + } + for (const [id, view] of running) { + if (!tracked.has(id)) { + void follow(id, view.name, view.project); + } + } +} + +async function sampleHost(at: number): Promise { + const reading = await readHost(); + const traffic = { ...reading.network, ...reading.io }; + const before = previousHost; + const seconds = before ? (at - before.at) / 1000 : 0; + previousHost = { at, idle: reading.times.idle, total: reading.times.total, traffic }; + return { + cpu: before ? Math.max(0, 100 * (1 - (reading.times.idle - before.idle) / Math.max(1, reading.times.total - before.total))) : 0, + cpus: reading.cpus, + load: reading.load, + uptime: reading.uptime, + memory: reading.memory, + swap: reading.swap, + disk: space, + rx: rate(traffic.rx, before?.traffic.rx, seconds), + tx: rate(traffic.tx, before?.traffic.tx, seconds), + read: rate(traffic.read, before?.traffic.read, seconds), + write: rate(traffic.write, before?.traffic.write, seconds), + }; +} + +function add(target: string, cpu: number, memory: number, traffic: Traffic) { + const entry = sums.get(target) ?? { seconds: 0, cpu: 0, cpuPeak: 0, memory: 0, memoryPeak: 0, net: 0, io: 0 }; + entry.seconds += 1; + entry.cpu += cpu; + entry.cpuPeak = Math.max(entry.cpuPeak, cpu); + entry.memory += memory; + entry.memoryPeak = Math.max(entry.memoryPeak, memory); + entry.net += traffic.rx + traffic.tx; + entry.io += traffic.read + traffic.write; + sums.set(target, entry); +} + +export function accumulate(history: Pick, frame: Frame) { + const start = Math.floor(frame.at / 60_000) * 60_000; + if (start !== minute && sums.size) { + history.record( + minute, + [...sums].map(([target, entry]) => ({ + target, + cpu: entry.cpu / entry.seconds, + cpuPeak: entry.cpuPeak, + memory: Math.round(entry.memory / entry.seconds), + memoryPeak: entry.memoryPeak, + net: entry.net / entry.seconds, + io: entry.io / entry.seconds, + })), + ); + sums.clear(); + } + minute = start; + add('host', frame.host.cpu, frame.host.memory.used, frame.host); + for (const [name, container] of Object.entries(frame.containers)) { + add(name, container.cpu, container.memory, container); + } +} + +export function latest() { + return frames.at(-1) ?? first; +} + +export function start(history: History) { + let busy = false; + const tick = async () => { + if (busy) { + return; + } + busy = true; + try { + const at = Date.now(); + const frame: Frame = { + at, + host: await sampleHost(at), + containers: Object.fromEntries([...tracked.values()].flatMap((entry) => (entry.value ? [[entry.name, entry.value]] : []))), + }; + frames.push(frame); + if (frames.length > windowSeconds) { + frames.shift(); + } + ready(frame); + accumulate(history, frame); + for (const listener of listeners) { + listener(frame); + } + } finally { + busy = false; + } + }; + const report = (task: string) => (error: unknown) => console.error(`${task} failed:`, error); + const measureSpace = () => diskSpace().then((value) => (space = value)); + void measureSpace().catch(report('disk space')); + const timers = [setInterval(() => void tick().catch(report('live sample')), 1000), setInterval(() => void measureSpace().catch(report('disk space')), 30_000)]; + return () => timers.forEach(clearInterval); +} + +export function serve(request: http.IncomingMessage, response: http.ServerResponse, allowed: () => Promise) { + response.writeHead(200, { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-store', 'X-Accel-Buffering': 'no' }); + response.write(`retry: 3000\nevent: backlog\ndata: ${JSON.stringify(frames)}\n\n`); + const send = (frame: Frame) => response.write(`event: frame\ndata: ${JSON.stringify(frame)}\n\n`); + listeners.add(send); + const check = setInterval(() => void allowed().then((still) => still || response.end(), () => response.end()), 60_000); + request.on('close', () => { + listeners.delete(send); + clearInterval(check); + }); +} diff --git a/app/src/server/main.ts b/app/src/server/main.ts index fea3cef..dd01b10 100644 --- a/app/src/server/main.ts +++ b/app/src/server/main.ts @@ -3,7 +3,7 @@ import http from 'node:http'; import path from 'node:path'; import { WebSocketServer } from 'ws'; import { Access, permits, scopePattern } from './access.ts'; -import { Alerts } from './alerts.ts'; +import { Alerts, ruleKinds, type RuleKind } from './alerts.ts'; import { authenticate, callback, githubToken, login, logout, organisationMembers, roleNow, sessionMs, type Role, type User } from './auth.ts'; import { download, listBackups, order, requests, upload } from './backups.ts'; import { config } from './config.ts'; @@ -11,12 +11,13 @@ import { docker } from './docker.ts'; import { docs } from './docs.ts'; import { History, projectTotals, ranges, type Range } from './history.ts'; import { collect, containers, diskUsage, projects, refresh, refreshDisk, snapshot, type ContainerView } from './inventory.ts'; -import { monitor } from './monitor.ts'; +import { serve as serveLive } from './live.ts'; +import { email, emailing, monitor } from './monitor.ts'; import { open, openSocket, parseOpen } from './proxy.ts'; import { Refusal } from './refusal.ts'; import { appliedAt, appliedCompose, appliedValueNames, redeployStatus, startRedeploy } from './rollout.ts'; import { composeReferences, parseKeys, projectPattern, SecretStore } from './secrets.ts'; -import { downloadLogs, logs, terminal } from './terminal.ts'; +import { downloadLogs, logs, searchLogs, terminal } from './terminal.ts'; const history = new History(config.dataDir); const access = new Access(config.dataDir); @@ -243,7 +244,43 @@ async function api(user: User, request: http.IncomingMessage, url: URL) { return { ...user, grants: user.owner ? ['*'] : access.scopes(user.login), ssh: user.owner ? config.sshTarget : null }; } if (route === 'GET /api/alerts') { - return { active: alerts.active(), recent: alerts.recent(), notifying: Boolean(config.alertsToken), repo: config.alertsRepo }; + return { + active: alerts.active(), + recent: alerts.recent(), + notifying: Boolean(config.alertsToken), + emailing: emailing(), + repo: config.alertsRepo, + rules: Object.values(alerts.rules()).map((rule) => ({ ...ruleKinds[rule.kind], ...rule })), + }; + } + const rule = /^POST \/api\/alerts\/rules\/([a-z-]+)$/.exec(route); + if (rule) { + await requireOwnerNow(request); + if (!(rule[1] in ruleKinds)) { + throw new Refusal(404, `There is no ${rule[1]} rule.`); + } + const { threshold, minutes, enabled } = (await body(request)) as { threshold?: unknown; minutes?: unknown; enabled?: unknown }; + const number = (value: unknown, most: number) => typeof value === 'number' && Number.isFinite(value) && value >= 0 && value <= most; + if (!number(threshold, 100_000) || !number(minutes, 1440) || typeof enabled !== 'boolean') { + throw new Refusal(400, 'A rule needs a threshold, minutes between 0 and 1440, and enabled.'); + } + alerts.setRule({ kind: rule[1] as RuleKind, threshold: threshold as number, minutes: minutes as number, enabled }); + return { ok: true }; + } + if (route === 'POST /api/alerts/test') { + await requireOwnerNow(request); + await email('Test alert from the Graphite console', `${user.login} sent this test from ${config.appUrl}/#/alerts.`); + return { ok: true }; + } + if (route === 'GET /api/logs/search') { + const text = (url.searchParams.get('q') ?? '').trim(); + const window = { '15m': 900_000, '1h': 3600_000, '6h': 21_600_000, '24h': 86_400_000, '7d': 604_800_000 }[url.searchParams.get('since') ?? '1h']; + if (text.length < 2 || !window) { + throw new Refusal(400, 'Search for at least two characters over 15m, 1h, 6h, 24h or 7d.'); + } + const project = url.searchParams.get('project'); + const chosen = (await containers()).filter((container) => !project || container.project === project); + return searchLogs(chosen, text, Date.now() - window); } if (route === 'GET /api/docs') { return docs(); @@ -429,6 +466,10 @@ async function handle(request: http.IncomingMessage, response: http.ServerRespon await requireNow(request, user, `project:${container.project}`); return open(request, response, target, url.search); } + if (url.pathname === '/api/live' && request.method === 'GET') { + const token = githubToken(request); + return serveLive(request, response, async () => (await roleNow(token)) !== undefined); + } const logFile = /^\/api\/containers\/([0-9a-f]{12,64})\/logs$/.exec(url.pathname); if (logFile && request.method === 'GET') { return downloadLogs(response, await findContainer(logFile[1])); diff --git a/app/src/server/monitor.ts b/app/src/server/monitor.ts index c1a469e..9d30874 100644 --- a/app/src/server/monitor.ts +++ b/app/src/server/monitor.ts @@ -1,38 +1,48 @@ -import type { Alert, Alerts, Condition } from './alerts.ts'; +import nodemailer from 'nodemailer'; +import { defaultRules, type Alert, type Alerts, type Condition, type Rule, type RuleKind } from './alerts.ts'; import { backupConditions } from './backups.ts'; import { config } from './config.ts'; import { docker } from './docker.ts'; import { snapshot, type ContainerView } from './inventory.ts'; +import { latest, type Frame, type HostLive } from './live.ts'; +import { Refusal } from './refusal.ts'; interface Inspect { RestartCount: number; State: { OOMKilled: boolean }; } -type Host = Awaited>['host']; +type Rules = Record; const restarts = new Map(); +const mailer = config.smtpUrl ? nodemailer.createTransport(config.smtpUrl) : undefined; + +const duration = (rule: Rule) => rule.minutes * 60_000; +const ratio = (usage: { used: number; total: number }) => (usage.total > 0 ? (100 * usage.used) / usage.total : 0); +const megabytes = (value: number) => `${Math.round(value / 1048576)} MB`; export function completed(container: ContainerView) { return container.state === 'exited' && container.status.startsWith('Exited (0)') && !container.sleeps; } -export function hostConditions(host: Host): Condition[] { +export function hostConditions(host: Pick, rules: Rules = defaultRules): Condition[] { const checks = [ - { key: 'memory', label: 'memory', usage: host.memory, limit: 0.9 }, - { key: 'swap', label: 'swap', usage: host.swap, limit: 0.8 }, - { key: 'disk', label: 'disk', usage: host.disk, limit: 0.85 }, - ]; + { kind: 'host-cpu', key: 'cpu', label: 'CPU', percent: host.cpu, detail: `across ${host.cpus} cores.` }, + { kind: 'host-memory', key: 'memory', label: 'memory', percent: ratio(host.memory), detail: `${megabytes(host.memory.used)} of ${megabytes(host.memory.total)}.` }, + { kind: 'host-swap', key: 'swap', label: 'swap', percent: ratio(host.swap), detail: `${megabytes(host.swap.used)} of ${megabytes(host.swap.total)}.` }, + { kind: 'host-disk', key: 'disk', label: 'disk', percent: ratio(host.disk), detail: `${megabytes(host.disk.used)} of ${megabytes(host.disk.total)}.` }, + ] as const; return checks - .filter((check) => check.usage.total > 0 && check.usage.used / check.usage.total > check.limit) + .filter((check) => rules[check.kind].enabled && check.percent > rules[check.kind].threshold) .map((check) => ({ key: check.key, - title: `The VPS ${check.label} is ${Math.round((100 * check.usage.used) / check.usage.total)}% used`, - detail: `Over ${Math.round(check.limit * 100)}%: ${Math.round(check.usage.used / 1048576)} MB of ${Math.round(check.usage.total / 1048576)} MB.`, + title: `The VPS ${check.label} is ${Math.round(check.percent)}% used`, + detail: `Over ${rules[check.kind].threshold}%: ${check.detail}`, + for: duration(rules[check.kind]), })); } -export function containerConditions(views: ContainerView[], inspects: Map, now: number): Condition[] { +export function containerConditions(views: ContainerView[], inspects: Map, now: number, rules: Rules = defaultRules): Condition[] { const found: Condition[] = []; for (const container of views) { if (completed(container) || (container.sleeps && container.state !== 'running')) { @@ -40,16 +50,21 @@ export function containerConditions(views: ContainerView[], inspects: Map entry.at > now - 10 * 60_000); restarts.set(container.name, history); const recent = inspect.RestartCount - history[0].count; - if (recent >= 3) { + if (rules['container-restarts'].enabled && recent >= rules['container-restarts'].threshold) { found.push({ key: `restarts:${container.name}`, title: `${container.name} keeps restarting`, detail: `${recent} restarts in the last 10 minutes.${oom}` }); } } @@ -57,8 +72,31 @@ export function containerConditions(views: ContainerView[], inspects: Map, rules: Rules = defaultRules): Condition[] { + return Object.entries(frame.containers).flatMap(([name, container]) => { + const cores = container.cpuLimit ?? frame.host.cpus; + const cpu = container.cpu / cores; + const memory = container.memoryLimit ? (100 * container.memory) / container.memoryLimit : 0; + return [ + ...(rules['container-cpu'].enabled && cpu > rules['container-cpu'].threshold + ? [{ key: `cpu:${name}`, title: `${name} uses ${Math.round(cpu)}% of its CPU`, detail: `Over ${rules['container-cpu'].threshold}% of ${cores} cores.`, for: duration(rules['container-cpu']) }] + : []), + ...(rules['container-memory'].enabled && memory > rules['container-memory'].threshold + ? [ + { + key: `memory:${name}`, + title: `${name} uses ${Math.round(memory)}% of its memory`, + detail: `${megabytes(container.memory)} of its ${megabytes(container.memoryLimit!)} limit.`, + for: duration(rules['container-memory']), + }, + ] + : []), + ]; + }); +} + +async function conditions(rules: Rules, now: number) { + const [{ host, views }, frame] = await Promise.all([snapshot(), latest()]); const inspects = new Map( await Promise.all( views @@ -66,7 +104,12 @@ async function conditions(now: number) { .map(async (container) => [container.id, await docker('GET', `/containers/${container.id}/json`)] as const), ), ); - return [...hostConditions(host), ...containerConditions(views, inspects, now), ...(await backupConditions(now))]; + return [ + ...hostConditions(host, rules), + ...containerConditions(views, inspects, now, rules), + ...usageConditions(frame, rules), + ...(rules['backup-late'].enabled ? await backupConditions(now, undefined, rules['backup-late'].threshold) : []), + ]; } async function github(method: string, route: string, body: unknown) { @@ -86,7 +129,7 @@ async function github(method: string, route: string, body: unknown) { return (await response.json()) as { number: number }; } -async function notify(alerts: Alerts, resolved: Alert[]) { +async function issues(alerts: Alerts, resolved: Alert[]) { for (const alert of alerts.active().filter((candidate) => candidate.issue === null)) { const issue = await github('POST', '/issues', { title: `Alert: ${alert.title}`, @@ -101,13 +144,47 @@ async function notify(alerts: Alerts, resolved: Alert[]) { } } -export function monitor(alerts: Alerts, every = 60_000) { +export const emailing = () => Boolean(mailer && config.alertsEmail); + +export async function email(subject: string, text: string) { + if (!mailer || !config.alertsEmail) { + throw new Refusal(503, 'Email needs SMTP_URL and ALERTS_EMAIL in the console env file.'); + } + await mailer.sendMail({ from: config.alertsFrom ?? config.alertsEmail, to: config.alertsEmail, subject, text }); +} + +async function emails(opened: Alert[], resolved: Alert[]) { + for (const alert of opened) { + await email(`Alert: ${alert.title}`, `${alert.detail}\n\nSince ${new Date(alert.since).toISOString()}.\n${config.appUrl}/#/alerts`); + } + for (const alert of resolved) { + await email(`Resolved: ${alert.title}`, `Resolved at ${new Date(alert.resolved!).toISOString()} after starting ${new Date(alert.since).toISOString()}.\n${config.appUrl}/#/alerts`); + } +} + +export function monitor(alerts: Alerts, every = 10_000) { const tick = async () => { - const { resolved } = alerts.update(await conditions(Date.now())); - if (config.alertsToken) { - await notify(alerts, resolved); + const now = Date.now(); + const { opened, resolved } = alerts.update(await conditions(alerts.rules(), now), now); + const sent = await Promise.allSettled([ + ...(config.alertsToken ? [issues(alerts, resolved)] : []), + ...(emailing() && (opened.length || resolved.length) ? [emails(opened, resolved)] : []), + ]); + for (const result of sent) { + if (result.status === 'rejected') { + console.error('alert notification failed:', result.reason); + } } }; - const timer = setInterval(() => void tick().catch((error: unknown) => console.error('alerts failed:', error)), every); + let running = false; + const timer = setInterval(() => { + if (running) { + return; + } + running = true; + void tick() + .catch((error: unknown) => console.error('alerts failed:', error)) + .finally(() => (running = false)); + }, every); return () => clearInterval(timer); } diff --git a/app/src/server/terminal.ts b/app/src/server/terminal.ts index 4d1187f..d2bb554 100644 --- a/app/src/server/terminal.ts +++ b/app/src/server/terminal.ts @@ -1,7 +1,7 @@ import type http from 'node:http'; import type { WebSocket } from 'ws'; import { config } from './config.ts'; -import { Demuxer, docker, LogLines, stream, upgrade } from './docker.ts'; +import { Demuxer, docker, LogLines, stream, upgrade, type LogLine } from './docker.ts'; const shell = ['/bin/sh', '-c', 'if command -v bash >/dev/null; then exec bash; else exec sh; fi']; const hostShell = ['nsenter', '--target', '1', '--all', '--', '/bin/bash', '-l']; @@ -88,6 +88,27 @@ export async function logs(socket: WebSocket, container: string, params: URLSear }); } +export async function searchLogs(containers: { id: string; name: string }[], text: string, since: number, limit = 500) { + const needle = text.toLowerCase(); + const found: (LogLine & { container: string })[] = []; + await Promise.all( + containers.map(async (container) => { + const { output, read } = await readLogs(container.id, `follow=0&tail=20000&since=${Math.floor(since / 1000)}`); + let matches = 0; + for await (const chunk of output) { + for (const line of read(chunk as Buffer)) { + if (matches < limit && line.text.toLowerCase().includes(needle)) { + found.push({ ...line, container: container.name }); + matches += 1; + } + } + } + }), + ); + found.sort((a, b) => b.time.localeCompare(a.time)); + return { lines: found.slice(0, limit), more: found.length > limit }; +} + export async function downloadLogs(response: http.ServerResponse, container: { id: string; name: string }) { const { output, read } = await readLogs(container.id, 'tail=all'); response.writeHead(200, { diff --git a/app/src/web/App.tsx b/app/src/web/App.tsx index dfb5bce..49f3ad3 100644 --- a/app/src/web/App.tsx +++ b/app/src/web/App.tsx @@ -11,6 +11,7 @@ import { ContainerPage } from './pages/ContainerPage'; import { Docs } from './pages/Docs'; import { HostShell } from './pages/HostShell'; import { Images } from './pages/Images'; +import { LogSearch } from './pages/LogSearch'; import { Overview } from './pages/Overview'; import { ProjectPage } from './pages/ProjectPage'; import { SecretPage, Secrets } from './pages/Secrets'; @@ -48,7 +49,8 @@ function Page({ hash, user }: { hash: string; user: User }) { if (user.owner && hash === '/backups') return ; if (hash === '/images') return ; if (hash === '/volumes') return ; - if (hash === '/alerts') return ; + if (hash === '/alerts') return ; + if (hash === '/logs') return ; if (can(user, 'area:shell') && hash === '/shell') return ; if (user.owner && hash === '/admin') return ; return ; @@ -69,8 +71,9 @@ export function App() { }, []); const pages: ShellTab[] = [ - { href: '#/', label: 'Overview', icon: 'home', active: !/^\/(images|volumes|alerts|shell|admin|docs|secrets|backups)/.test(hash) }, + { href: '#/', label: 'Overview', icon: 'home', active: !/^\/(images|volumes|alerts|logs|shell|admin|docs|secrets|backups)/.test(hash) }, { href: '#/alerts', label: alerts ? `Alerts · ${alerts}` : 'Alerts', icon: 'bell', active: hash === '/alerts' }, + { href: '#/logs', label: 'Logs', icon: 'search', active: hash === '/logs' }, { href: '#/images', label: 'Images', icon: 'library', active: hash === '/images' }, { href: '#/volumes', label: 'Volumes', icon: 'folder', active: hash === '/volumes' }, ...(user && can(user, 'area:shell') ? ([{ href: '#/shell', label: 'Shell', icon: 'monitor', active: hash === '/shell' }] satisfies ShellTab[]) : []), diff --git a/app/src/web/api.ts b/app/src/web/api.ts index 0e894f6..667e69b 100644 --- a/app/src/web/api.ts +++ b/app/src/web/api.ts @@ -137,7 +137,38 @@ export interface SecretDetail extends Secret { export interface Point { at: number; cpu: number; + cpuPeak: number; memory: number; + memoryPeak: number; + net: number; + io: number; +} + +export type Metric = Exclude; + +export interface Traffic { + rx: number; + tx: number; + read: number; + write: number; +} + +export interface ContainerLive extends Traffic { + project: string; + cpu: number; + cpuLimit: number | null; + memory: number; + memoryLimit: number | null; +} + +export interface Frame { + at: number; + host: Host & Traffic; + containers: Record; +} + +export function rate(value: number | null | undefined) { + return value === null || value === undefined ? '—' : `${bytes(value)}/s`; } export interface Series { @@ -180,11 +211,29 @@ export interface Alert { issue: number | null; } +export interface AlertRule { + kind: string; + label: string; + unit: string | null; + threshold: number; + minutes: number; + enabled: boolean; +} + export interface AlertsData { active: Alert[]; recent: Alert[]; notifying: boolean; + emailing: boolean; repo: string; + rules: AlertRule[]; +} + +export interface LogMatch { + container: string; + stream: 'stdout' | 'stderr'; + time: string; + text: string; } export interface DocPage { diff --git a/app/src/web/components/Chart.tsx b/app/src/web/components/Chart.tsx index a2a7161..e18efba 100644 --- a/app/src/web/components/Chart.tsx +++ b/app/src/web/components/Chart.tsx @@ -1,5 +1,5 @@ import { useEffect, useRef, useState, type PointerEvent } from 'react'; -import type { Series } from '../api'; +import type { Metric, Series } from '../api'; const palette = ['#6a9eea', '#5fbf7f', '#e0915a', '#a388e8', '#4fb8b0', '#e07fb5', '#d7b24a', '#e5736b', '#8b8b95']; const height = 180; @@ -8,7 +8,7 @@ const pad = { top: 10, right: 10, bottom: 22, left: 58 }; export interface ChartProps { label: string; series: Series[]; - metric: 'cpu' | 'memory'; + metric: Metric; span: number; step: number; now: number; @@ -30,6 +30,7 @@ function timeLabel(at: number, span: number, precise = false) { return new Intl.DateTimeFormat(undefined, { ...(span > day || precise ? { day: 'numeric', month: 'short' } : {}), ...(span <= 7 * day || precise ? { hour: '2-digit', minute: '2-digit' } : {}), + ...(span <= 600_000 ? { second: '2-digit' } : {}), }).format(at); } @@ -44,7 +45,7 @@ function useWidth() { return { ref, width }; } -function Breakdown({ lines, at, metric, total, format }: { lines: Series[]; at: number; metric: 'cpu' | 'memory'; total: number; format: (value: number) => string }) { +function Breakdown({ lines, at, metric, total, format }: { lines: Series[]; at: number; metric: Metric; total: number; format: (value: number) => string }) { const parts = lines .map((line) => ({ name: line.name, value: line.points.find((point) => point.at === at)?.[metric] ?? 0 })) .filter((part) => part.value > 0) diff --git a/app/src/web/components/Metrics.tsx b/app/src/web/components/Metrics.tsx index f0fb4d5..73c6e82 100644 --- a/app/src/web/components/Metrics.tsx +++ b/app/src/web/components/Metrics.tsx @@ -1,36 +1,59 @@ import { useState } from 'react'; -import { bytes, percent, ranges, type HistoryData, type Range } from '../api'; +import { bytes, percent, rate, ranges, type HistoryData, type Metric, type Series } from '../api'; +import { containerPoint, hostPoint, liveSeconds, useLive } from '../live'; import { load, save } from '../stored'; import { usePoll } from '../usePoll'; import { Chart } from './Chart'; import { Segmented } from './Segmented'; -export function Metrics({ query, cpuMax, memoryMax }: { query: string; cpuMax?: number; memoryMax?: number }) { - const [range, setRange] = useState(() => { +const options = ['live', ...ranges] as const; +type Option = (typeof options)[number]; + +export type LiveTargets = 'host' | { name: string; label: string }[]; + +export function Metrics({ query, live, cpuMax, memoryMax }: { query: string; live: LiveTargets; cpuMax?: number; memoryMax?: number }) { + const [range, setRange] = useState