From aa63347de575af244bd15357a93957431fadf94d Mon Sep 17 00:00:00 2001 From: Amr Mohammed El-Sheraey <141947355+AmrMsCLL@users.noreply.github.com> Date: Thu, 5 Nov 2026 04:58:24 +0200 Subject: [PATCH 1/3] fix: redeploy through a host unit, an honest image prune, accurate disk use and stricter requests Redeploy runs graphite-up as a transient systemd unit through nsenter instead of calling the agent, returns at once, and the page polls the unit's state and log, so Cloudflare's 100 second limit and a redeploy of the console itself both work. The missing-secret hint ignores names the project's values file supplies, and the needs-redeploy hint counts graphite-up's .applied marker as a rollout. Removing images now removes every unused image and refreshes disk use. Disk use is reported as df does. Every response carries the security headers, and a write needs this console's Origin or Sec-Fetch-Site same-origin. --- app/src/server/config.ts | 1 - app/src/server/host.ts | 2 +- app/src/server/main.ts | 73 +++++++++++++++--------- app/src/server/rollout.ts | 88 +++++++++++++++++++++++++++++ app/src/server/secrets.ts | 11 ---- app/src/web/components/Redeploy.tsx | 61 ++++++++++++++++++++ app/src/web/pages/Images.tsx | 15 +++-- app/src/web/pages/ProjectPage.tsx | 10 +++- app/src/web/pages/Secrets.tsx | 23 +------- 9 files changed, 212 insertions(+), 72 deletions(-) create mode 100644 app/src/server/rollout.ts create mode 100644 app/src/web/components/Redeploy.tsx diff --git a/app/src/server/config.ts b/app/src/server/config.ts index df29d3d..855a955 100644 --- a/app/src/server/config.ts +++ b/app/src/server/config.ts @@ -29,6 +29,5 @@ export const config = { sshTarget: process.env.SSH_TARGET ?? 'ubuntu@localhost', secretsKey: process.env.SECRETS_KEY || undefined, graphiteHome: process.env.GRAPHITE_HOME ?? '/srv/graphite', - agentUrl: process.env.AGENT_URL ?? 'http://172.17.0.1:9000', webRoot: new URL('../web/', import.meta.url), }; diff --git a/app/src/server/host.ts b/app/src/server/host.ts index 13d87f0..dc2e951 100644 --- a/app/src/server/host.ts +++ b/app/src/server/host.ts @@ -39,7 +39,7 @@ export async function hostStats() { cpus: os.cpus().length, load: load.split(' ').slice(0, 3).map(Number), uptime: Number(uptime.split(' ')[0]), - disk: { used: (disk.blocks - disk.bfree) * disk.bsize, total: disk.blocks * disk.bsize }, + disk: { used: (disk.blocks - disk.bfree) * disk.bsize, total: (disk.blocks - disk.bfree + disk.bavail) * disk.bsize }, ...meminfo(memory), }; } diff --git a/app/src/server/main.ts b/app/src/server/main.ts index e3b2a50..fea3cef 100644 --- a/app/src/server/main.ts +++ b/app/src/server/main.ts @@ -10,11 +10,12 @@ import { config } from './config.ts'; import { docker } from './docker.ts'; import { docs } from './docs.ts'; import { History, projectTotals, ranges, type Range } from './history.ts'; -import { collect, containers, diskUsage, projects, refresh, snapshot, type ContainerView } from './inventory.ts'; +import { collect, containers, diskUsage, projects, refresh, refreshDisk, snapshot, type ContainerView } from './inventory.ts'; import { monitor } from './monitor.ts'; import { open, openSocket, parseOpen } from './proxy.ts'; import { Refusal } from './refusal.ts'; -import { appliedCompose, composeReferences, parseKeys, projectPattern, SecretStore } from './secrets.ts'; +import { appliedAt, appliedCompose, appliedValueNames, redeployStatus, startRedeploy } from './rollout.ts'; +import { composeReferences, parseKeys, projectPattern, SecretStore } from './secrets.ts'; import { downloadLogs, logs, terminal } from './terminal.ts'; const history = new History(config.dataDir); @@ -143,18 +144,19 @@ function secretStore() { return store; } -function needsRedeploy(project: string, views: ContainerView[]) { +async function needsRedeploy(project: string, views: ContainerView[]) { const changed = store?.changedAt(project) ?? null; const created = views.filter((view) => view.project === project).map((view) => view.created); - return changed !== null && created.length > 0 && changed > Math.max(...created); + return changed !== null && created.length > 0 && changed > Math.max(...created, (await appliedAt(project)) ?? 0); } async function secretProject(project: string, views: ContainerView[]) { + const [compose, values] = await Promise.all([appliedCompose(project), appliedValueNames(project)]); const stored = new Set(secretStore().list(project).map((secret) => secret.name)); return { project, - needsRedeploy: needsRedeploy(project, views), - missing: [...composeReferences(await appliedCompose(project))].filter(([name, reference]) => reference.required && !stored.has(name)).map(([name]) => name), + needsRedeploy: await needsRedeploy(project, views), + missing: [...composeReferences(compose)].filter(([name, reference]) => reference.required && !stored.has(name) && !values.has(name)).map(([name]) => name), }; } @@ -166,7 +168,7 @@ async function secretsApi(user: User, request: http.IncomingMessage, route: stri const slugs = new Set([...secrets.projects(), ...views.map((view) => view.project)].filter((slug) => projectPattern.test(slug) && slug !== 'console' && slug !== 'other')); return { projects: await Promise.all([...slugs].sort().map((slug) => secretProject(slug, views))), secrets: secrets.list() }; } - const match = /^(GET|POST) \/api\/secrets\/([a-z0-9-]+)(?:\/([A-Z_][A-Z0-9_]*|import|delete|redeploy))?(?:\/(reveal|restore))?$/.exec(route); + const match = /^(GET|POST) \/api\/secrets\/([a-z0-9-]+)(?:\/([A-Z_][A-Z0-9_]*|import|delete))?(?:\/(reveal|restore))?$/.exec(route); if (!match) { throw new Refusal(404, 'Not found.'); } @@ -205,21 +207,6 @@ async function secretsApi(user: User, request: http.IncomingMessage, route: stri secrets.remove(project, names, user.login); return { ok: true }; } - if (target === 'redeploy' && !action) { - await requireNow(request, user, `project:${project}`); - const response = await fetch(`${config.agentUrl}/reapply`, { - method: 'POST', - headers: { Authorization: `Bearer ${secrets.value('agent', 'CONSOLE_TOKEN')}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ name: project, by: user.login }), - signal: AbortSignal.timeout(660_000), - }); - if (!response.ok) { - throw new Refusal(502, `The agent answered ${response.status}: ${await response.text()}`); - } - secrets.audit(user.login, 'redeploy', project, null); - await refresh(); - return { ok: true }; - } if (!action) { secrets.update(project, target, await body(request), user.login); return { ok: true }; @@ -273,7 +260,7 @@ async function api(user: User, request: http.IncomingMessage, url: URL) { buildCache: usage.buildCache, volumes: usage.volumes.reduce((sum, volume) => sum + Math.max(0, volume.size), 0), }, - projects: list.map((project) => ({ ...project, needsRedeploy: needsRedeploy(project.slug, project.containers) })), + projects: await Promise.all(list.map(async (project) => ({ ...project, needsRedeploy: await needsRedeploy(project.slug, project.containers) }))), }; } const project = /^GET \/api\/projects\/([a-z0-9-]+)$/.exec(route); @@ -340,8 +327,12 @@ async function api(user: User, request: http.IncomingMessage, url: URL) { } if (route === 'POST /api/prune') { await requireNow(request, user, 'area:images'); - const result = await docker<{ SpaceReclaimed: number }>('POST', `/images/prune?filters=${encodeURIComponent('{"dangling":["true"]}')}`); - return { reclaimed: result.SpaceReclaimed }; + const result = await docker<{ SpaceReclaimed: number; ImagesDeleted: { Deleted?: string }[] | null }>( + 'POST', + `/images/prune?filters=${encodeURIComponent('{"dangling":["false"]}')}`, + ); + await refreshDisk(); + return { reclaimed: result.SpaceReclaimed, removed: (result.ImagesDeleted ?? []).filter((entry) => entry.Deleted).length }; } const action = /^POST \/api\/containers\/([0-9a-f]{12,64})\/(start|stop|restart)$/.exec(route); if (action) { @@ -351,6 +342,16 @@ async function api(user: User, request: http.IncomingMessage, url: URL) { await refresh(); return { ok: true }; } + const redeploy = /^(GET|POST) \/api\/projects\/([a-z0-9-]+)\/redeploy$/.exec(route); + if (redeploy) { + if (redeploy[1] === 'GET') { + return redeployStatus(redeploy[2]); + } + await requireNow(request, user, `project:${redeploy[2]}`); + await startRedeploy(redeploy[2]); + store?.audit(user.login, 'redeploy', redeploy[2], null); + return { ok: true }; + } const wake = /^POST \/api\/projects\/([a-z0-9-]+)\/wake$/.exec(route); if (wake) { await requireNow(request, user, `project:${wake[1]}`); @@ -375,8 +376,6 @@ async function serveFile(response: http.ServerResponse, pathname: string) { 'Content-Type': types[extension] ?? 'application/octet-stream', 'Cache-Control': name.startsWith('assets/') ? 'public, max-age=31536000, immutable' : 'no-cache', 'Content-Security-Policy': policy, - 'X-Content-Type-Options': 'nosniff', - 'Referrer-Policy': 'same-origin', }); response.end(file); } catch { @@ -384,8 +383,26 @@ async function serveFile(response: http.ServerResponse, pathname: string) { } } +const headers = { + 'X-Content-Type-Options': 'nosniff', + 'X-Frame-Options': 'DENY', + 'Referrer-Policy': 'same-origin', + 'Cross-Origin-Opener-Policy': 'same-origin', + 'Cross-Origin-Resource-Policy': 'same-origin', + 'Permissions-Policy': 'camera=(), microphone=(), geolocation=(), payment=(), usb=()', + ...(config.production ? { 'Strict-Transport-Security': 'max-age=63072000; includeSubDomains' } : {}), +}; + +function sameOrigin(request: http.IncomingMessage) { + const origin = request.headers.origin; + return origin ? origin === new URL(config.appUrl).origin : request.headers['sec-fetch-site'] === 'same-origin'; +} + async function handle(request: http.IncomingMessage, response: http.ServerResponse) { const url = new URL(request.url ?? '/', config.appUrl); + for (const [name, value] of Object.entries(headers)) { + response.setHeader(name, value); + } if (url.pathname === '/healthz') { response.writeHead(200).end('ok'); return; @@ -422,7 +439,7 @@ async function handle(request: http.IncomingMessage, response: http.ServerRespon return download(response, archive[1], archive[2], user.login); } if (url.pathname.startsWith('/api/')) { - if (request.method === 'POST' && request.headers.origin && request.headers.origin !== new URL(config.appUrl).origin) { + if (request.method !== 'GET' && !sameOrigin(request)) { throw new Refusal(403, 'Cross-origin requests are refused.'); } const importing = /^\/api\/backups\/([a-z0-9][a-z0-9_-]*)\/import$/.exec(url.pathname); diff --git a/app/src/server/rollout.ts b/app/src/server/rollout.ts new file mode 100644 index 0000000..d92d6d8 --- /dev/null +++ b/app/src/server/rollout.ts @@ -0,0 +1,88 @@ +import { readFile, stat } from 'node:fs/promises'; +import path from 'node:path'; +import { config } from './config.ts'; +import { Demuxer, docker, stream } from './docker.ts'; +import { Refusal } from './refusal.ts'; + +function applied(project: string, file: string) { + return path.join(config.hostRoot, config.graphiteHome, project, file); +} + +function missingIsEmpty(error: NodeJS.ErrnoException) { + if (error.code === 'ENOENT') { + return undefined; + } + throw error; +} + +export async function appliedCompose(project: string) { + return (await readFile(applied(project, 'compose.yaml'), 'utf8').catch(missingIsEmpty)) ?? ''; +} + +export async function appliedValueNames(project: string) { + const text = (await readFile(applied(project, 'values'), 'utf8').catch(missingIsEmpty)) ?? ''; + return new Set(text.split('\n').flatMap((line) => /^([A-Z_][A-Z0-9_]*)=/.exec(line)?.[1] ?? [])); +} + +export async function appliedAt(project: string) { + return (await stat(applied(project, '.applied')).catch(missingIsEmpty))?.mtimeMs ?? null; +} + +export async function onHost(command: string[]) { + const { Id } = await docker<{ Id: string }>('POST', `/containers/${config.self}/exec`, { + AttachStdout: true, + AttachStderr: true, + Tty: false, + Cmd: ['nsenter', '--target', '1', '--all', '--', ...command], + }); + const demuxer = new Demuxer(); + const chunks: Buffer[] = []; + for await (const chunk of await stream('POST', `/exec/${Id}/start`, { Detach: false, Tty: false })) { + chunks.push(...demuxer.push(chunk as Buffer).map((frame) => frame.data)); + } + const { ExitCode } = await docker<{ ExitCode: number }>('GET', `/exec/${Id}/json`); + return { code: ExitCode, output: Buffer.concat(chunks).toString('utf8').trim() }; +} + +const unit = (project: string) => `graphite-redeploy-${project}`; + +export async function redeployStatus(project: string) { + const { output } = await onHost(['systemctl', 'show', unit(project), '--property=ActiveState,SubState,InvocationID,StateChangeTimestamp']); + const properties = Object.fromEntries(output.split('\n').map((line) => [line.slice(0, line.indexOf('=')), line.slice(line.indexOf('=') + 1)])); + const state = + properties.ActiveState === 'activating' || properties.SubState === 'running' + ? 'running' + : properties.ActiveState === 'failed' + ? 'failed' + : properties.SubState === 'exited' + ? 'done' + : 'none'; + const log = + state !== 'none' && properties.InvocationID + ? (await onHost(['journalctl', `_SYSTEMD_INVOCATION_ID=${properties.InvocationID}`, '--no-pager', '--output=cat', '--lines=30'])).output + : ''; + return { state, log, at: properties.StateChangeTimestamp || null }; +} + +export async function startRedeploy(project: string) { + if ((await stat(applied(project, 'compose.yaml')).catch(missingIsEmpty)) === undefined) { + throw new Refusal(404, `${project} has not been deployed by the agent.`); + } + if ((await redeployStatus(project)).state === 'running') { + throw new Refusal(409, `${project} is already redeploying.`); + } + await onHost(['systemctl', 'reset-failed', unit(project)]); + await onHost(['systemctl', 'stop', unit(project)]); + const started = await onHost([ + 'systemd-run', + `--unit=${unit(project)}`, + `--description=Redeploy ${project} from the console`, + '--remain-after-exit', + '--property=RuntimeMaxSec=900', + path.posix.join(config.graphiteHome, 'bin', 'graphite-up'), + project, + ]); + if (started.code !== 0) { + throw new Refusal(502, `The host refused to start the redeploy: ${started.output}`); + } +} diff --git a/app/src/server/secrets.ts b/app/src/server/secrets.ts index e9f1e51..e3e3384 100644 --- a/app/src/server/secrets.ts +++ b/app/src/server/secrets.ts @@ -1,9 +1,7 @@ import { createCipheriv, createDecipheriv, createHash, randomBytes } from 'node:crypto'; import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, statSync, writeFileSync } from 'node:fs'; -import { readFile } from 'node:fs/promises'; import path from 'node:path'; import { DatabaseSync } from 'node:sqlite'; -import { config } from './config.ts'; import { Refusal } from './refusal.ts'; export const projectPattern = /^[a-z0-9](?:[a-z0-9-]{0,38}[a-z0-9])?$/; @@ -130,15 +128,6 @@ export function composeReferences(text: string) { return references; } -export async function appliedCompose(project: string) { - return readFile(path.join(config.hostRoot, config.graphiteHome, project, 'compose.yaml'), 'utf8').catch((error: NodeJS.ErrnoException) => { - if (error.code === 'ENOENT') { - return ''; - } - throw error; - }); -} - export class SecretStore { private readonly db: DatabaseSync; diff --git a/app/src/web/components/Redeploy.tsx b/app/src/web/components/Redeploy.tsx new file mode 100644 index 0000000..00d0403 --- /dev/null +++ b/app/src/web/components/Redeploy.tsx @@ -0,0 +1,61 @@ +import { Button } from '@project-graphite/ui'; +import { useEffect, useState } from 'react'; +import { get, post } from '../api'; + +interface Status { + state: 'none' | 'running' | 'done' | 'failed'; + log: string; +} + +export function Redeploy({ project, onDone }: { project: string; onDone: () => void }) { + const [watching, setWatching] = useState(false); + const [status, setStatus] = useState(); + const [message, setMessage] = useState(); + + useEffect(() => { + if (!watching) { + return; + } + const timer = setInterval(() => { + get(`/api/projects/${project}/redeploy`).then( + (next) => { + setStatus(next); + setMessage(undefined); + if (next.state === 'done' || next.state === 'failed') { + setWatching(false); + onDone(); + } + }, + () => setMessage('Waiting for the console to answer…'), + ); + }, 3000); + return () => clearInterval(timer); + }, [watching, project, onDone]); + + const start = () => { + setStatus(undefined); + setMessage('Starting…'); + post(`/api/projects/${project}/redeploy`).then( + () => { + setMessage(undefined); + setStatus({ state: 'running', log: '' }); + setWatching(true); + }, + (error: Error) => setMessage(error.message), + ); + }; + + return ( + + + + + {message ?? (status?.state === 'running' ? 'Redeploying…' : status?.state === 'done' ? 'Redeployed.' : status?.state === 'failed' ? 'The redeploy failed:' : '')} + + + {status?.state === 'failed' && status.log &&
{status.log}
} +
+ ); +} diff --git a/app/src/web/pages/Images.tsx b/app/src/web/pages/Images.tsx index 9a21ef5..23c2596 100644 --- a/app/src/web/pages/Images.tsx +++ b/app/src/web/pages/Images.tsx @@ -7,7 +7,7 @@ export function Images({ user }: { user: User }) { const images = usePoll('/api/images', 30_000); const disk = usePoll('/api/disk', 30_000); const [pruning, setPruning] = useState(false); - const [reclaimed, setReclaimed] = useState(); + const [result, setResult] = useState<{ reclaimed: number; removed: number }>(); if (images.error && !images.data) { return {images.error.message}; @@ -19,7 +19,7 @@ export function Images({ user }: { user: User }) { setPruning(true)}>Remove dangling images + ) } title="Images" @@ -27,7 +27,7 @@ export function Images({ user }: { user: User }) { {disk.data && ( images {bytes(disk.data.images)} · containers {bytes(disk.data.containers)} · build cache {bytes(disk.data.buildCache)} - {reclaimed !== undefined && ` · reclaimed ${bytes(reclaimed)}`} + {result && ` · removed ${result.removed} images, reclaimed ${bytes(result.reclaimed)}`} )} @@ -59,13 +59,12 @@ export function Images({ user }: { user: User }) { confirmLabel="Remove" onClose={() => setPruning(false)} onConfirm={async () => { - const result = await post<{ reclaimed: number }>('/api/prune'); - setReclaimed(result.reclaimed); - await images.reload(); + setResult(await post<{ reclaimed: number; removed: number }>('/api/prune')); + await Promise.all([images.reload(), disk.reload()]); }} - title="Remove dangling images?" + title={`Remove ${unused.length} unused images?`} > - Removes untagged image layers no container uses. Tagged images stay. + Removes every image no container uses, running or stopped, older builds included. A project pulls its image again when it next deploys. )} diff --git a/app/src/web/pages/ProjectPage.tsx b/app/src/web/pages/ProjectPage.tsx index a0ff15f..8bfbb75 100644 --- a/app/src/web/pages/ProjectPage.tsx +++ b/app/src/web/pages/ProjectPage.tsx @@ -3,6 +3,7 @@ import { Button, ErrorState, PageHeader, TagChip } from '@project-graphite/ui'; import { useState } from 'react'; import { bytes, can, percent, post, shortImage, stateColor, stateOf, type ProjectDetail, type User } from '../api'; import { Logs } from '../components/Logs'; +import { Redeploy } from '../components/Redeploy'; import { Metrics } from '../components/Metrics'; import { usePoll } from '../usePoll'; import { SecretList } from './Secrets'; @@ -39,7 +40,14 @@ export function ProjectPage({ slug, user }: { slug: string; user: User }) { return (
void reload()} slug={slug} />} + actions={ + can(user, `project:${slug}`) && ( + + void reload()} project={slug} /> + {project.sleeps && void reload()} slug={slug} />} + + ) + } eyebrow={Overview} title={slug} > diff --git a/app/src/web/pages/Secrets.tsx b/app/src/web/pages/Secrets.tsx index 2f47ee0..aa6bec4 100644 --- a/app/src/web/pages/Secrets.tsx +++ b/app/src/web/pages/Secrets.tsx @@ -18,6 +18,7 @@ import { } from '@project-graphite/ui'; import { useEffect, useState } from 'react'; import { can, post, type Secret, type SecretDetail, type SecretProject, type SecretType, type User } from '../api'; +import { Redeploy } from '../components/Redeploy'; import { Segmented } from '../components/Segmented'; import { usePoll } from '../usePoll'; @@ -28,28 +29,6 @@ const typeOptions: SelectOption[] = [ const when = (at: number) => timeAgo(new Date(at).toISOString()); -function Redeploy({ project, onDone }: { project: string; onDone: () => void }) { - const [message, setMessage] = useState(); - const redeploy = () => { - setMessage('Redeploying…'); - post(`/api/secrets/${project}/redeploy`).then( - () => { - setMessage('Redeployed.'); - onDone(); - }, - (error: Error) => setMessage(error.message), - ); - }; - return ( - - - {message && {message}} - - ); -} - function SecretSheet({ prefix, secret, onClose, onSaved }: { prefix: string; secret?: Secret; onClose: () => void; onSaved: () => void }) { const [mode, setMode] = useState<'Create' | 'Import'>('Create'); const [path, setPath] = useState(secret ? `/${secret.project}/${secret.name}` : prefix); From 9f1a46c3626ce57395e1bf2f2f69e14d76cd52be Mon Sep 17 00:00:00 2001 From: Amr Mohamed El-Sheraey <141947355+AmrMsCLL@users.noreply.github.com> Date: Thu, 5 Nov 2026 13:06:06 +0200 Subject: [PATCH 2/3] feat: live monitoring every second, minute averages and peaks, editable alert rules with email, and log search (#21) A stats stream per running container and /proc on the host feed a 1 second frame: CPU, memory, network and disk rates. GET /api/live streams it to every tab over Server-Sent Events with the last three minutes, and closes when the viewer stops being a member. History keeps each minute's average and peak, with network and disk, in the existing table. Charts gain a live range, network and disk, and a peak view. Alerts are checked every 10 seconds against rules owners edit (threshold, duration, on or off), open once a condition holds for its duration, resolve after a clear minute, and are emailed through SMTP_URL to ALERTS_EMAIL with a test button. Logs adds a search across containers. --- app/package-lock.json | 21 +++ app/package.json | 2 + app/src/server/alerts.ts | 76 +++++++-- app/src/server/backups.ts | 4 +- app/src/server/config.ts | 3 + app/src/server/docker.ts | 17 ++ app/src/server/history.ts | 28 +++- app/src/server/host.ts | 44 +++-- app/src/server/inventory.ts | 67 +++----- app/src/server/live.ts | 244 ++++++++++++++++++++++++++++ app/src/server/main.ts | 49 +++++- app/src/server/monitor.ts | 127 ++++++++++++--- app/src/server/terminal.ts | 23 ++- app/src/web/App.tsx | 7 +- app/src/web/api.ts | 49 ++++++ app/src/web/components/Chart.tsx | 7 +- app/src/web/components/Metrics.tsx | 57 +++++-- app/src/web/live.ts | 65 ++++++++ app/src/web/pages/Alerts.tsx | 118 +++++++++++++- app/src/web/pages/ContainerPage.tsx | 2 +- app/src/web/pages/LogSearch.tsx | 73 +++++++++ app/src/web/pages/Overview.tsx | 46 ++++-- app/src/web/pages/ProjectPage.tsx | 2 +- app/test/alerts.spec.ts | 73 +++++++-- app/test/history.spec.ts | 72 +++++--- app/test/host.spec.ts | 23 ++- app/test/live.spec.ts | 42 +++++ 27 files changed, 1147 insertions(+), 194 deletions(-) create mode 100644 app/src/server/live.ts create mode 100644 app/src/web/live.ts create mode 100644 app/src/web/pages/LogSearch.tsx create mode 100644 app/test/live.spec.ts diff --git a/app/package-lock.json b/app/package-lock.json index 2972a15..be1ce05 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -15,6 +15,7 @@ "@xterm/addon-fit": "0.11.0", "@xterm/xterm": "6.0.0", "marked": "18.0.14", + "nodemailer": "10.3.1", "react": "19.3.0", "react-dom": "19.3.0", "ws": "8.22.0" @@ -22,6 +23,7 @@ "devDependencies": { "@tailwindcss/vite": "4.3.3", "@types/node": "26.6.5", + "@types/nodemailer": "8.0.2", "@types/react": "19.3.0", "@types/react-dom": "19.3.0", "@types/ws": "8.18.2", @@ -1497,6 +1499,16 @@ "undici-types": "~8.9.0" } }, + "node_modules/@types/nodemailer": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@types/nodemailer/-/nodemailer-8.0.2.tgz", + "integrity": "sha512-c7M5ox8p0nEOfbJ2E9Qcmt8/QCu/VzsiAhzBiZbvky2PhKZDHpKB3sQHBM5MEZNkCfBOaS6S9//AGPRTNB7IaA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/parse-json": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz", @@ -2317,6 +2329,15 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/nodemailer": { + "version": "10.3.1", + "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-10.3.1.tgz", + "integrity": "sha512-uF0AFsJC07Y1vdW7p580DKAnh7rAO+4UPSSB3mDCPNDufoiaOebV6JmrzpxB14OjEEEiBsNdYoM2Y+fN8M7BBA==", + "license": "MIT-0", + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", diff --git a/app/package.json b/app/package.json index 69a5c28..83d5263 100644 --- a/app/package.json +++ b/app/package.json @@ -18,6 +18,7 @@ "@xterm/addon-fit": "0.11.0", "@xterm/xterm": "6.0.0", "marked": "18.0.14", + "nodemailer": "10.3.1", "react": "19.3.0", "react-dom": "19.3.0", "ws": "8.22.0" @@ -25,6 +26,7 @@ "devDependencies": { "@tailwindcss/vite": "4.3.3", "@types/node": "26.6.5", + "@types/nodemailer": "8.0.2", "@types/react": "19.3.0", "@types/react-dom": "19.3.0", "@types/ws": "8.18.2", diff --git a/app/src/server/alerts.ts b/app/src/server/alerts.ts index 0969cf3..3e11927 100644 --- a/app/src/server/alerts.ts +++ b/app/src/server/alerts.ts @@ -6,22 +6,49 @@ export interface Condition { key: string; title: string; detail: string; + for?: number; } -export interface Alert extends Condition { +export interface Alert extends Omit { id: number; since: number; resolved: number | null; issue: number | null; } -const confirmAfter = 2; +export const ruleKinds = { + 'host-cpu': { label: 'Host CPU above', unit: '%', threshold: 90, minutes: 5 }, + 'host-memory': { label: 'Host memory above', unit: '%', threshold: 90, minutes: 2 }, + 'host-swap': { label: 'Swap above', unit: '%', threshold: 80, minutes: 2 }, + 'host-disk': { label: 'Disk above', unit: '%', threshold: 85, minutes: 2 }, + 'container-cpu': { label: 'A container above its CPU limit by', unit: '%', threshold: 90, minutes: 5 }, + 'container-memory': { label: 'A container above its memory limit by', unit: '%', threshold: 90, minutes: 2 }, + 'container-down': { label: 'A container stopped', unit: null, threshold: 0, minutes: 1 }, + 'container-unhealthy': { label: 'A container unhealthy', unit: null, threshold: 0, minutes: 1 }, + 'container-restarts': { label: 'A container restarted in 10 minutes, times', unit: 'restarts', threshold: 3, minutes: 0 }, + 'backup-late': { label: 'A project without a backup for', unit: 'hours', threshold: 26, minutes: 0 }, +} as const; + +export type RuleKind = keyof typeof ruleKinds; + +export interface Rule { + kind: RuleKind; + threshold: number; + minutes: number; + enabled: boolean; +} + +export const defaultRules = Object.fromEntries( + Object.entries(ruleKinds).map(([kind, rule]) => [kind, { kind, threshold: rule.threshold, minutes: rule.minutes, enabled: true }]), +) as Record; + +const resolveAfter = 60_000; const keepDays = 30; export class Alerts { private readonly db: DatabaseSync; - private readonly present = new Map(); - private readonly absent = new Map(); + private readonly since = new Map(); + private readonly clear = new Map(); constructor(directory: string) { mkdirSync(directory, { recursive: true }); @@ -32,9 +59,25 @@ export class Alerts { since INTEGER NOT NULL, resolved INTEGER, issue INTEGER ); CREATE INDEX IF NOT EXISTS alerts_open ON alerts (key) WHERE resolved IS NULL; + CREATE TABLE IF NOT EXISTS rules (kind TEXT PRIMARY KEY, threshold REAL NOT NULL, minutes REAL NOT NULL, enabled INTEGER NOT NULL); `); } + rules(): Record { + const saved = this.db.prepare('SELECT kind, threshold, minutes, enabled FROM rules').all() as { kind: string; threshold: number; minutes: number; enabled: number }[]; + const rules = structuredClone(defaultRules); + for (const row of saved.filter((entry) => entry.kind in ruleKinds)) { + rules[row.kind as RuleKind] = { kind: row.kind as RuleKind, threshold: row.threshold, minutes: row.minutes, enabled: row.enabled === 1 }; + } + return rules; + } + + setRule(rule: Rule) { + this.db + .prepare('INSERT INTO rules (kind, threshold, minutes, enabled) VALUES (?, ?, ?, ?) ON CONFLICT (kind) DO UPDATE SET threshold = excluded.threshold, minutes = excluded.minutes, enabled = excluded.enabled') + .run(rule.kind, rule.threshold, rule.minutes, rule.enabled ? 1 : 0); + } + active() { return this.db.prepare('SELECT * FROM alerts WHERE resolved IS NULL ORDER BY since').all() as unknown as Alert[]; } @@ -51,35 +94,34 @@ export class Alerts { const opened: Alert[] = []; const resolved: Alert[] = []; for (const condition of conditions) { - this.absent.delete(condition.key); - const streak = (this.present.get(condition.key) ?? 0) + 1; - this.present.set(condition.key, streak); + this.clear.delete(condition.key); + const first = this.since.get(condition.key) ?? now; + this.since.set(condition.key, first); const alert = open.get(condition.key); if (alert) { this.db.prepare('UPDATE alerts SET title = ?, detail = ? WHERE id = ?').run(condition.title, condition.detail, alert.id); - } else if (streak >= confirmAfter) { + } else if (now - first >= (condition.for ?? 0)) { const { lastInsertRowid } = this.db .prepare('INSERT INTO alerts (key, title, detail, since) VALUES (?, ?, ?, ?)') - .run(condition.key, condition.title, condition.detail, now); - opened.push({ ...condition, id: Number(lastInsertRowid), since: now, resolved: null, issue: null }); + .run(condition.key, condition.title, condition.detail, first); + opened.push({ key: condition.key, title: condition.title, detail: condition.detail, id: Number(lastInsertRowid), since: first, resolved: null, issue: null }); } } for (const alert of open.values()) { if (seen.has(alert.key)) { continue; } - this.present.delete(alert.key); - const streak = (this.absent.get(alert.key) ?? 0) + 1; - this.absent.set(alert.key, streak); - if (streak >= confirmAfter) { + const cleared = this.clear.get(alert.key) ?? now; + this.clear.set(alert.key, cleared); + if (now - cleared >= resolveAfter) { this.db.prepare('UPDATE alerts SET resolved = ? WHERE id = ?').run(now, alert.id); - this.absent.delete(alert.key); + this.clear.delete(alert.key); resolved.push({ ...alert, resolved: now }); } } - for (const key of this.present.keys()) { + for (const key of this.since.keys()) { if (!seen.has(key)) { - this.present.delete(key); + this.since.delete(key); } } this.db.prepare('DELETE FROM alerts WHERE resolved IS NOT NULL AND resolved < ?').run(now - keepDays * 86400_000); diff --git a/app/src/server/backups.ts b/app/src/server/backups.ts index a1f81b6..15fefe6 100644 --- a/app/src/server/backups.ts +++ b/app/src/server/backups.ts @@ -79,13 +79,13 @@ export function stampTime(stamp: string) { return Date.parse(`${stamp.slice(0, 4)}-${stamp.slice(4, 6)}-${stamp.slice(6, 8)}T${stamp.slice(9, 11)}:${stamp.slice(11, 13)}:${stamp.slice(13, 15)}Z`); } -export async function backupConditions(now: number, root = backupRoot()) { +export async function backupConditions(now: number, root = backupRoot(), hours = 26) { return (await listBackups(root)).flatMap(({ project, archives }) => { const latest = archives.find((archive) => !archive.imported); if (!latest) { return []; } - if (now - stampTime(latest.stamp) > 26 * hour) { + if (now - stampTime(latest.stamp) > hours * hour) { return [{ key: `backup:${project}`, title: `${project} has no recent backup`, detail: `The newest backup is ${Math.round((now - stampTime(latest.stamp)) / hour)} hours old.` }]; } if (!latest.ok) { diff --git a/app/src/server/config.ts b/app/src/server/config.ts index 855a955..8d22d33 100644 --- a/app/src/server/config.ts +++ b/app/src/server/config.ts @@ -20,6 +20,9 @@ export const config = { organisation: process.env.GITHUB_ORG ?? 'Project-Graphite', alertsToken: process.env.ALERTS_TOKEN || undefined, alertsRepo: process.env.ALERTS_REPO ?? 'project-graphite/platform', + smtpUrl: process.env.SMTP_URL || undefined, + alertsEmail: process.env.ALERTS_EMAIL || undefined, + alertsFrom: process.env.ALERTS_FROM || undefined, docsRepo: process.env.DOCS_REPO ?? 'project-graphite/docs', dockerSocket: process.env.DOCKER_SOCKET ?? '/var/run/docker.sock', hostRoot: process.env.HOST_ROOT ?? '/proc/1/root', diff --git a/app/src/server/docker.ts b/app/src/server/docker.ts index ff4c8cf..3b57ca8 100644 --- a/app/src/server/docker.ts +++ b/app/src/server/docker.ts @@ -16,9 +16,26 @@ export interface DockerContainer { } export interface DockerStats { + read?: string; cpu_stats: { cpu_usage: { total_usage: number }; system_cpu_usage?: number; online_cpus?: number }; precpu_stats: { cpu_usage: { total_usage: number }; system_cpu_usage?: number }; memory_stats: { usage?: number; limit?: number; stats?: { inactive_file?: number } }; + networks?: Record; + blkio_stats?: { io_service_bytes_recursive?: { op: string; value: number }[] | null }; +} + +export function networkTotals(stats: DockerStats) { + return Object.values(stats.networks ?? {}).reduce((sum, network) => ({ rx: sum.rx + network.rx_bytes, tx: sum.tx + network.tx_bytes }), { rx: 0, tx: 0 }); +} + +export function ioTotals(stats: DockerStats) { + return (stats.blkio_stats?.io_service_bytes_recursive ?? []).reduce( + (sum, entry) => ({ + read: sum.read + (entry.op.toLowerCase() === 'read' ? entry.value : 0), + write: sum.write + (entry.op.toLowerCase() === 'write' ? entry.value : 0), + }), + { read: 0, write: 0 }, + ); } function call(method: string, path: string, body?: unknown, headers: Record = {}) { diff --git a/app/src/server/history.ts b/app/src/server/history.ts index f0fd6b8..37db658 100644 --- a/app/src/server/history.ts +++ b/app/src/server/history.ts @@ -21,13 +21,21 @@ export const keep = ranges['30d'].span; export interface Sample { target: string; cpu: number; + cpuPeak: number; memory: number; + memoryPeak: number; + net: number; + io: number; } export interface Point { at: number; cpu: number; + cpuPeak: number; memory: number; + memoryPeak: number; + net: number; + io: number; } export function projectTotals(containers: { name: string; project: string }[], seriesOf: (name: string) => Point[], cpus: number) { @@ -36,9 +44,13 @@ export function projectTotals(containers: { name: string; project: string }[], s const totals = projects.get(container.project) ?? new Map(); projects.set(container.project, totals); for (const point of seriesOf(container.name)) { - const total = totals.get(point.at) ?? { at: point.at, cpu: 0, memory: 0 }; + const total = totals.get(point.at) ?? { at: point.at, cpu: 0, cpuPeak: 0, memory: 0, memoryPeak: 0, net: 0, io: 0 }; total.cpu += point.cpu / cpus; + total.cpuPeak += point.cpuPeak / cpus; total.memory += point.memory; + total.memoryPeak += point.memoryPeak; + total.net += point.net; + total.io += point.io; totals.set(point.at, total); } } @@ -58,12 +70,18 @@ export class History { CREATE INDEX IF NOT EXISTS samples_target_at ON samples (target, at); CREATE INDEX IF NOT EXISTS samples_at ON samples (at); `); + const columns = new Set((this.db.prepare('PRAGMA table_info(samples)').all() as { name: string }[]).map((column) => column.name)); + for (const [name, type] of [['cpu_peak', 'REAL'], ['memory_peak', 'INTEGER'], ['net', 'REAL'], ['io', 'REAL']]) { + if (!columns.has(name)) { + this.db.exec(`ALTER TABLE samples ADD COLUMN ${name} ${type}`); + } + } } record(at: number, samples: Sample[]) { - const insert = this.db.prepare('INSERT INTO samples (at, target, cpu, memory) VALUES (?, ?, ?, ?)'); + const insert = this.db.prepare('INSERT INTO samples (at, target, cpu, cpu_peak, memory, memory_peak, net, io) VALUES (?, ?, ?, ?, ?, ?, ?, ?)'); for (const sample of samples) { - insert.run(at, sample.target, sample.cpu, sample.memory); + insert.run(at, sample.target, sample.cpu, sample.cpuPeak, sample.memory, sample.memoryPeak, sample.net, sample.io); } this.db.prepare('DELETE FROM samples WHERE at < ?').run(at - keep); } @@ -72,7 +90,9 @@ export class History { const { span, step } = ranges[range]; return this.db .prepare( - `SELECT CAST(at / ? AS INTEGER) * ? AS at, avg(cpu) AS cpu, CAST(avg(memory) AS INTEGER) AS memory + `SELECT CAST(at / ? AS INTEGER) * ? AS at, avg(cpu) AS cpu, max(coalesce(cpu_peak, cpu)) AS cpuPeak, + CAST(avg(memory) AS INTEGER) AS memory, max(coalesce(memory_peak, memory)) AS memoryPeak, + coalesce(avg(net), 0) AS net, coalesce(avg(io), 0) AS io FROM samples WHERE target = ? AND at > ? GROUP BY 1 ORDER BY 1`, ) .all(step, step, target, now - span) as unknown as Point[]; diff --git a/app/src/server/host.ts b/app/src/server/host.ts index dc2e951..2aa3cf3 100644 --- a/app/src/server/host.ts +++ b/app/src/server/host.ts @@ -2,8 +2,6 @@ import { readFile, statfs } from 'node:fs/promises'; import os from 'node:os'; import { config } from './config.ts'; -let previous: { idle: number; total: number } | undefined; - export function cpuTimes(stat: string) { const fields = stat.split('\n')[0].trim().split(/\s+/).slice(1).map(Number); const idle = fields[3] + (fields[4] ?? 0); @@ -23,23 +21,39 @@ export function meminfo(text: string) { }; } -export async function hostStats() { - const [stat, memory, load, uptime, disk] = await Promise.all([ - readFile('/proc/stat', 'utf8'), - readFile('/proc/meminfo', 'utf8'), - readFile('/proc/loadavg', 'utf8'), - readFile('/proc/uptime', 'utf8'), - statfs(config.hostRoot).catch(() => statfs('/')), - ]); - const times = cpuTimes(stat); - const cpu = previous ? 100 * (1 - (times.idle - previous.idle) / Math.max(1, times.total - previous.total)) : 0; - previous = times; +export function networkBytes(text: string) { + return text + .split('\n') + .slice(2) + .map((line) => line.trim().split(/[:\s]+/)) + .filter((fields) => fields.length > 9 && !/^(lo|veth|br-|docker)/.test(fields[0])) + .reduce((sum, fields) => ({ rx: sum.rx + Number(fields[1]), tx: sum.tx + Number(fields[9]) }), { rx: 0, tx: 0 }); +} + +export function diskBytes(text: string) { + return text + .split('\n') + .map((line) => line.trim().split(/\s+/)) + .filter((fields) => /^(sd[a-z]+|vd[a-z]+|xvd[a-z]+|nvme\d+n\d+)$/.test(fields[2] ?? '')) + .reduce((sum, fields) => ({ read: sum.read + Number(fields[5]) * 512, write: sum.write + Number(fields[9]) * 512 }), { read: 0, write: 0 }); +} + +export async function readHost() { + const [stat, memory, load, uptime, network, disks] = await Promise.all( + ['/proc/stat', '/proc/meminfo', '/proc/loadavg', '/proc/uptime', '/proc/1/net/dev', '/proc/diskstats'].map((file) => readFile(file, 'utf8')), + ); return { - cpu: Math.max(0, cpu), + times: cpuTimes(stat), cpus: os.cpus().length, load: load.split(' ').slice(0, 3).map(Number), uptime: Number(uptime.split(' ')[0]), - disk: { used: (disk.blocks - disk.bfree) * disk.bsize, total: (disk.blocks - disk.bfree + disk.bavail) * disk.bsize }, ...meminfo(memory), + network: networkBytes(network), + io: diskBytes(disks), }; } + +export async function diskSpace() { + const disk = await statfs(config.hostRoot).catch(() => statfs('/')); + return { used: (disk.blocks - disk.bfree) * disk.bsize, total: (disk.blocks - disk.bfree + disk.bavail) * disk.bsize }; +} diff --git a/app/src/server/inventory.ts b/app/src/server/inventory.ts index feccb30..1bb55f3 100644 --- a/app/src/server/inventory.ts +++ b/app/src/server/inventory.ts @@ -1,6 +1,6 @@ -import { cpuPercent, docker, memoryUsed, type DockerContainer, type DockerStats } from './docker.ts'; +import { docker, type DockerContainer } from './docker.ts'; import type { History } from './history.ts'; -import { hostStats } from './host.ts'; +import { latest, start, track, type ContainerLive, type HostLive } from './live.ts'; export interface ContainerView { id: string; @@ -29,10 +29,8 @@ export interface Disk { volumes: { name: string; project: string | null; size: number; inUse: boolean }[]; } -type Snapshot = { host: Awaited>; views: ContainerView[] }; - -let latest: Snapshot | undefined; -let measuring: Promise | undefined; +let listed: ContainerView[] | undefined; +let measuring: Promise | undefined; let disk: Disk | undefined; let reading: Promise | undefined; @@ -40,7 +38,7 @@ function health(status: string) { return /\((healthy|unhealthy|health: starting)\)/.exec(status)?.[1] ?? null; } -function view(container: DockerContainer, stats?: DockerStats): ContainerView { +function view(container: DockerContainer): ContainerView { return { id: container.Id, name: container.Names[0].replace(/^\//, ''), @@ -55,33 +53,30 @@ function view(container: DockerContainer, stats?: DockerStats): ContainerView { ports: [...new Set(container.Ports.map((port) => port.PrivatePort))].sort((a, b) => a - b), networks: Object.fromEntries(Object.entries(container.NetworkSettings.Networks).map(([name, network]) => [name, network.IPAddress])), sleeps: container.Labels['sablier.enable'] === 'true', - cpu: stats ? cpuPercent(stats) : null, - memory: stats ? memoryUsed(stats) : null, - memoryLimit: stats?.memory_stats.limit ?? null, + cpu: null, + memory: null, + memoryLimit: null, }; } -async function measure() { - const containers = await docker('GET', '/containers/json?all=1'); - const stats = await Promise.all( - containers.map((container) => - container.State === 'running' - ? docker('GET', `/containers/${container.Id}/stats?stream=false`).catch(() => undefined) - : undefined, - ), - ); - return containers.map((container, index) => view(container, stats[index])); +export function withLive(view: ContainerView, live: ContainerLive | undefined): ContainerView { + return view.state === 'running' && live ? { ...view, cpu: live.cpu, memory: live.memory, memoryLimit: live.memoryLimit } : view; } export function refresh() { - measuring ??= Promise.all([hostStats(), measure()]) - .then(([host, views]) => (latest = { host, views })) + measuring ??= docker('GET', '/containers/json?all=1') + .then((containers) => { + listed = containers.map(view); + track(listed); + return listed; + }) .finally(() => (measuring = undefined)); return measuring; } -export async function snapshot() { - return latest ?? (await refresh()); +export async function snapshot(): Promise<{ host: HostLive; views: ContainerView[] }> { + const [views, frame] = await Promise.all([listed ?? refresh(), latest()]); + return { host: frame.host, views: views.map((entry) => withLive(entry, frame.containers[entry.name])) }; } export async function containers() { @@ -151,25 +146,17 @@ export function projects(views: ContainerView[], usage?: Disk) { }); } -export function collect(history: History, { measureEvery = 5000, recordEvery = 60_000, diskEvery = 300_000 } = {}) { - const record = async () => { - const { host, views } = await snapshot(); - history.record(Date.now(), [ - { target: 'host', cpu: host.cpu, memory: host.memory.used }, - ...views.filter((container) => container.cpu !== null).map((container) => ({ - target: container.name, - cpu: container.cpu!, - memory: container.memory!, - })), - ]); - }; +export function collect(history: History, { listEvery = 5000, diskEvery = 300_000 } = {}) { const report = (task: string) => (error: unknown) => console.error(`${task} failed:`, error); + const stop = start(history); const timers = [ - setInterval(() => void refresh().catch(report('measure')), measureEvery), - setInterval(() => void record().catch(report('sample')), recordEvery), + setInterval(() => void refresh().catch(report('container list')), listEvery), setInterval(() => void refreshDisk().catch(report('disk usage')), diskEvery), ]; - void refresh().catch(report('measure')); + void refresh().catch(report('container list')); void refreshDisk().catch(report('disk usage')); - return () => timers.forEach(clearInterval); + return () => { + stop(); + timers.forEach(clearInterval); + }; } diff --git a/app/src/server/live.ts b/app/src/server/live.ts new file mode 100644 index 0000000..3938ae6 --- /dev/null +++ b/app/src/server/live.ts @@ -0,0 +1,244 @@ +import type http from 'node:http'; +import { cpuPercent, docker, ioTotals, memoryUsed, networkTotals, stream, type DockerStats } from './docker.ts'; +import type { History } from './history.ts'; +import { diskSpace, readHost } from './host.ts'; + +export interface Traffic { + rx: number; + tx: number; + read: number; + write: number; +} + +export interface ContainerLive extends Traffic { + project: string; + cpu: number; + cpuLimit: number | null; + memory: number; + memoryLimit: number | null; +} + +export interface HostLive extends Traffic { + cpu: number; + cpus: number; + load: number[]; + uptime: number; + memory: { used: number; total: number }; + swap: { used: number; total: number }; + disk: { used: number; total: number }; +} + +export interface Frame { + at: number; + host: HostLive; + containers: Record; +} + +interface Tracked { + name: string; + project: string; + cpuLimit: number | null; + response?: http.IncomingMessage; + last?: Traffic & { at: number }; + value?: ContainerLive; +} + +interface Sums { + seconds: number; + cpu: number; + cpuPeak: number; + memory: number; + memoryPeak: number; + net: number; + io: number; +} + +export const windowSeconds = 180; +const tracked = new Map(); +const frames: Frame[] = []; +const listeners = new Set<(frame: Frame) => void>(); +const sums = new Map(); +let minute = 0; +let previousHost: { at: number; idle: number; total: number; traffic: Traffic } | undefined; +let space = { used: 0, total: 0 }; +let ready: (frame: Frame) => void; +const first = new Promise((resolve) => (ready = resolve)); + +export function rate(now: number, before: number | undefined, seconds: number) { + return before === undefined || seconds <= 0 ? 0 : Math.max(0, (now - before) / seconds); +} + +function read(entry: Tracked, stats: DockerStats) { + const at = Date.parse(stats.read ?? '') || Date.now(); + const traffic = { ...networkTotals(stats), ...ioTotals(stats) }; + const seconds = entry.last ? (at - entry.last.at) / 1000 : 0; + entry.value = { + project: entry.project, + cpu: cpuPercent(stats), + cpuLimit: entry.cpuLimit, + memory: memoryUsed(stats), + memoryLimit: stats.memory_stats.limit ?? null, + rx: rate(traffic.rx, entry.last?.rx, seconds), + tx: rate(traffic.tx, entry.last?.tx, seconds), + read: rate(traffic.read, entry.last?.read, seconds), + write: rate(traffic.write, entry.last?.write, seconds), + }; + entry.last = { at, ...traffic }; +} + +async function follow(id: string, name: string, project: string) { + const entry: Tracked = { name, project, cpuLimit: null }; + tracked.set(id, entry); + try { + const { HostConfig } = await docker<{ HostConfig: { NanoCpus: number } }>('GET', `/containers/${id}/json`); + entry.cpuLimit = HostConfig.NanoCpus ? HostConfig.NanoCpus / 1e9 : null; + const response = await stream('GET', `/containers/${id}/stats?stream=1`); + if (tracked.get(id) !== entry) { + response.destroy(); + return; + } + entry.response = response; + let buffer = ''; + response.setEncoding('utf8'); + response.on('data', (chunk: string) => { + buffer += chunk; + for (let end = buffer.indexOf('\n'); end >= 0; end = buffer.indexOf('\n')) { + const line = buffer.slice(0, end).trim(); + buffer = buffer.slice(end + 1); + if (line) { + read(entry, JSON.parse(line) as DockerStats); + } + } + }); + response.on('close', () => { + if (tracked.get(id) === entry) { + tracked.delete(id); + } + }); + } catch (error) { + tracked.delete(id); + console.error(`following ${name} failed:`, error); + } +} + +export function track(views: { id: string; name: string; project: string; state: string }[]) { + const running = new Map(views.filter((view) => view.state === 'running').map((view) => [view.id, view])); + for (const [id, entry] of tracked) { + if (!running.has(id)) { + entry.response?.destroy(); + tracked.delete(id); + } + } + for (const [id, view] of running) { + if (!tracked.has(id)) { + void follow(id, view.name, view.project); + } + } +} + +async function sampleHost(at: number): Promise { + const reading = await readHost(); + const traffic = { ...reading.network, ...reading.io }; + const before = previousHost; + const seconds = before ? (at - before.at) / 1000 : 0; + previousHost = { at, idle: reading.times.idle, total: reading.times.total, traffic }; + return { + cpu: before ? Math.max(0, 100 * (1 - (reading.times.idle - before.idle) / Math.max(1, reading.times.total - before.total))) : 0, + cpus: reading.cpus, + load: reading.load, + uptime: reading.uptime, + memory: reading.memory, + swap: reading.swap, + disk: space, + rx: rate(traffic.rx, before?.traffic.rx, seconds), + tx: rate(traffic.tx, before?.traffic.tx, seconds), + read: rate(traffic.read, before?.traffic.read, seconds), + write: rate(traffic.write, before?.traffic.write, seconds), + }; +} + +function add(target: string, cpu: number, memory: number, traffic: Traffic) { + const entry = sums.get(target) ?? { seconds: 0, cpu: 0, cpuPeak: 0, memory: 0, memoryPeak: 0, net: 0, io: 0 }; + entry.seconds += 1; + entry.cpu += cpu; + entry.cpuPeak = Math.max(entry.cpuPeak, cpu); + entry.memory += memory; + entry.memoryPeak = Math.max(entry.memoryPeak, memory); + entry.net += traffic.rx + traffic.tx; + entry.io += traffic.read + traffic.write; + sums.set(target, entry); +} + +export function accumulate(history: Pick, frame: Frame) { + const start = Math.floor(frame.at / 60_000) * 60_000; + if (start !== minute && sums.size) { + history.record( + minute, + [...sums].map(([target, entry]) => ({ + target, + cpu: entry.cpu / entry.seconds, + cpuPeak: entry.cpuPeak, + memory: Math.round(entry.memory / entry.seconds), + memoryPeak: entry.memoryPeak, + net: entry.net / entry.seconds, + io: entry.io / entry.seconds, + })), + ); + sums.clear(); + } + minute = start; + add('host', frame.host.cpu, frame.host.memory.used, frame.host); + for (const [name, container] of Object.entries(frame.containers)) { + add(name, container.cpu, container.memory, container); + } +} + +export function latest() { + return frames.at(-1) ?? first; +} + +export function start(history: History) { + let busy = false; + const tick = async () => { + if (busy) { + return; + } + busy = true; + try { + const at = Date.now(); + const frame: Frame = { + at, + host: await sampleHost(at), + containers: Object.fromEntries([...tracked.values()].flatMap((entry) => (entry.value ? [[entry.name, entry.value]] : []))), + }; + frames.push(frame); + if (frames.length > windowSeconds) { + frames.shift(); + } + ready(frame); + accumulate(history, frame); + for (const listener of listeners) { + listener(frame); + } + } finally { + busy = false; + } + }; + const report = (task: string) => (error: unknown) => console.error(`${task} failed:`, error); + const measureSpace = () => diskSpace().then((value) => (space = value)); + void measureSpace().catch(report('disk space')); + const timers = [setInterval(() => void tick().catch(report('live sample')), 1000), setInterval(() => void measureSpace().catch(report('disk space')), 30_000)]; + return () => timers.forEach(clearInterval); +} + +export function serve(request: http.IncomingMessage, response: http.ServerResponse, allowed: () => Promise) { + response.writeHead(200, { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-store', 'X-Accel-Buffering': 'no' }); + response.write(`retry: 3000\nevent: backlog\ndata: ${JSON.stringify(frames)}\n\n`); + const send = (frame: Frame) => response.write(`event: frame\ndata: ${JSON.stringify(frame)}\n\n`); + listeners.add(send); + const check = setInterval(() => void allowed().then((still) => still || response.end(), () => response.end()), 60_000); + request.on('close', () => { + listeners.delete(send); + clearInterval(check); + }); +} diff --git a/app/src/server/main.ts b/app/src/server/main.ts index fea3cef..dd01b10 100644 --- a/app/src/server/main.ts +++ b/app/src/server/main.ts @@ -3,7 +3,7 @@ import http from 'node:http'; import path from 'node:path'; import { WebSocketServer } from 'ws'; import { Access, permits, scopePattern } from './access.ts'; -import { Alerts } from './alerts.ts'; +import { Alerts, ruleKinds, type RuleKind } from './alerts.ts'; import { authenticate, callback, githubToken, login, logout, organisationMembers, roleNow, sessionMs, type Role, type User } from './auth.ts'; import { download, listBackups, order, requests, upload } from './backups.ts'; import { config } from './config.ts'; @@ -11,12 +11,13 @@ import { docker } from './docker.ts'; import { docs } from './docs.ts'; import { History, projectTotals, ranges, type Range } from './history.ts'; import { collect, containers, diskUsage, projects, refresh, refreshDisk, snapshot, type ContainerView } from './inventory.ts'; -import { monitor } from './monitor.ts'; +import { serve as serveLive } from './live.ts'; +import { email, emailing, monitor } from './monitor.ts'; import { open, openSocket, parseOpen } from './proxy.ts'; import { Refusal } from './refusal.ts'; import { appliedAt, appliedCompose, appliedValueNames, redeployStatus, startRedeploy } from './rollout.ts'; import { composeReferences, parseKeys, projectPattern, SecretStore } from './secrets.ts'; -import { downloadLogs, logs, terminal } from './terminal.ts'; +import { downloadLogs, logs, searchLogs, terminal } from './terminal.ts'; const history = new History(config.dataDir); const access = new Access(config.dataDir); @@ -243,7 +244,43 @@ async function api(user: User, request: http.IncomingMessage, url: URL) { return { ...user, grants: user.owner ? ['*'] : access.scopes(user.login), ssh: user.owner ? config.sshTarget : null }; } if (route === 'GET /api/alerts') { - return { active: alerts.active(), recent: alerts.recent(), notifying: Boolean(config.alertsToken), repo: config.alertsRepo }; + return { + active: alerts.active(), + recent: alerts.recent(), + notifying: Boolean(config.alertsToken), + emailing: emailing(), + repo: config.alertsRepo, + rules: Object.values(alerts.rules()).map((rule) => ({ ...ruleKinds[rule.kind], ...rule })), + }; + } + const rule = /^POST \/api\/alerts\/rules\/([a-z-]+)$/.exec(route); + if (rule) { + await requireOwnerNow(request); + if (!(rule[1] in ruleKinds)) { + throw new Refusal(404, `There is no ${rule[1]} rule.`); + } + const { threshold, minutes, enabled } = (await body(request)) as { threshold?: unknown; minutes?: unknown; enabled?: unknown }; + const number = (value: unknown, most: number) => typeof value === 'number' && Number.isFinite(value) && value >= 0 && value <= most; + if (!number(threshold, 100_000) || !number(minutes, 1440) || typeof enabled !== 'boolean') { + throw new Refusal(400, 'A rule needs a threshold, minutes between 0 and 1440, and enabled.'); + } + alerts.setRule({ kind: rule[1] as RuleKind, threshold: threshold as number, minutes: minutes as number, enabled }); + return { ok: true }; + } + if (route === 'POST /api/alerts/test') { + await requireOwnerNow(request); + await email('Test alert from the Graphite console', `${user.login} sent this test from ${config.appUrl}/#/alerts.`); + return { ok: true }; + } + if (route === 'GET /api/logs/search') { + const text = (url.searchParams.get('q') ?? '').trim(); + const window = { '15m': 900_000, '1h': 3600_000, '6h': 21_600_000, '24h': 86_400_000, '7d': 604_800_000 }[url.searchParams.get('since') ?? '1h']; + if (text.length < 2 || !window) { + throw new Refusal(400, 'Search for at least two characters over 15m, 1h, 6h, 24h or 7d.'); + } + const project = url.searchParams.get('project'); + const chosen = (await containers()).filter((container) => !project || container.project === project); + return searchLogs(chosen, text, Date.now() - window); } if (route === 'GET /api/docs') { return docs(); @@ -429,6 +466,10 @@ async function handle(request: http.IncomingMessage, response: http.ServerRespon await requireNow(request, user, `project:${container.project}`); return open(request, response, target, url.search); } + if (url.pathname === '/api/live' && request.method === 'GET') { + const token = githubToken(request); + return serveLive(request, response, async () => (await roleNow(token)) !== undefined); + } const logFile = /^\/api\/containers\/([0-9a-f]{12,64})\/logs$/.exec(url.pathname); if (logFile && request.method === 'GET') { return downloadLogs(response, await findContainer(logFile[1])); diff --git a/app/src/server/monitor.ts b/app/src/server/monitor.ts index c1a469e..9d30874 100644 --- a/app/src/server/monitor.ts +++ b/app/src/server/monitor.ts @@ -1,38 +1,48 @@ -import type { Alert, Alerts, Condition } from './alerts.ts'; +import nodemailer from 'nodemailer'; +import { defaultRules, type Alert, type Alerts, type Condition, type Rule, type RuleKind } from './alerts.ts'; import { backupConditions } from './backups.ts'; import { config } from './config.ts'; import { docker } from './docker.ts'; import { snapshot, type ContainerView } from './inventory.ts'; +import { latest, type Frame, type HostLive } from './live.ts'; +import { Refusal } from './refusal.ts'; interface Inspect { RestartCount: number; State: { OOMKilled: boolean }; } -type Host = Awaited>['host']; +type Rules = Record; const restarts = new Map(); +const mailer = config.smtpUrl ? nodemailer.createTransport(config.smtpUrl) : undefined; + +const duration = (rule: Rule) => rule.minutes * 60_000; +const ratio = (usage: { used: number; total: number }) => (usage.total > 0 ? (100 * usage.used) / usage.total : 0); +const megabytes = (value: number) => `${Math.round(value / 1048576)} MB`; export function completed(container: ContainerView) { return container.state === 'exited' && container.status.startsWith('Exited (0)') && !container.sleeps; } -export function hostConditions(host: Host): Condition[] { +export function hostConditions(host: Pick, rules: Rules = defaultRules): Condition[] { const checks = [ - { key: 'memory', label: 'memory', usage: host.memory, limit: 0.9 }, - { key: 'swap', label: 'swap', usage: host.swap, limit: 0.8 }, - { key: 'disk', label: 'disk', usage: host.disk, limit: 0.85 }, - ]; + { kind: 'host-cpu', key: 'cpu', label: 'CPU', percent: host.cpu, detail: `across ${host.cpus} cores.` }, + { kind: 'host-memory', key: 'memory', label: 'memory', percent: ratio(host.memory), detail: `${megabytes(host.memory.used)} of ${megabytes(host.memory.total)}.` }, + { kind: 'host-swap', key: 'swap', label: 'swap', percent: ratio(host.swap), detail: `${megabytes(host.swap.used)} of ${megabytes(host.swap.total)}.` }, + { kind: 'host-disk', key: 'disk', label: 'disk', percent: ratio(host.disk), detail: `${megabytes(host.disk.used)} of ${megabytes(host.disk.total)}.` }, + ] as const; return checks - .filter((check) => check.usage.total > 0 && check.usage.used / check.usage.total > check.limit) + .filter((check) => rules[check.kind].enabled && check.percent > rules[check.kind].threshold) .map((check) => ({ key: check.key, - title: `The VPS ${check.label} is ${Math.round((100 * check.usage.used) / check.usage.total)}% used`, - detail: `Over ${Math.round(check.limit * 100)}%: ${Math.round(check.usage.used / 1048576)} MB of ${Math.round(check.usage.total / 1048576)} MB.`, + title: `The VPS ${check.label} is ${Math.round(check.percent)}% used`, + detail: `Over ${rules[check.kind].threshold}%: ${check.detail}`, + for: duration(rules[check.kind]), })); } -export function containerConditions(views: ContainerView[], inspects: Map, now: number): Condition[] { +export function containerConditions(views: ContainerView[], inspects: Map, now: number, rules: Rules = defaultRules): Condition[] { const found: Condition[] = []; for (const container of views) { if (completed(container) || (container.sleeps && container.state !== 'running')) { @@ -40,16 +50,21 @@ export function containerConditions(views: ContainerView[], inspects: Map entry.at > now - 10 * 60_000); restarts.set(container.name, history); const recent = inspect.RestartCount - history[0].count; - if (recent >= 3) { + if (rules['container-restarts'].enabled && recent >= rules['container-restarts'].threshold) { found.push({ key: `restarts:${container.name}`, title: `${container.name} keeps restarting`, detail: `${recent} restarts in the last 10 minutes.${oom}` }); } } @@ -57,8 +72,31 @@ export function containerConditions(views: ContainerView[], inspects: Map, rules: Rules = defaultRules): Condition[] { + return Object.entries(frame.containers).flatMap(([name, container]) => { + const cores = container.cpuLimit ?? frame.host.cpus; + const cpu = container.cpu / cores; + const memory = container.memoryLimit ? (100 * container.memory) / container.memoryLimit : 0; + return [ + ...(rules['container-cpu'].enabled && cpu > rules['container-cpu'].threshold + ? [{ key: `cpu:${name}`, title: `${name} uses ${Math.round(cpu)}% of its CPU`, detail: `Over ${rules['container-cpu'].threshold}% of ${cores} cores.`, for: duration(rules['container-cpu']) }] + : []), + ...(rules['container-memory'].enabled && memory > rules['container-memory'].threshold + ? [ + { + key: `memory:${name}`, + title: `${name} uses ${Math.round(memory)}% of its memory`, + detail: `${megabytes(container.memory)} of its ${megabytes(container.memoryLimit!)} limit.`, + for: duration(rules['container-memory']), + }, + ] + : []), + ]; + }); +} + +async function conditions(rules: Rules, now: number) { + const [{ host, views }, frame] = await Promise.all([snapshot(), latest()]); const inspects = new Map( await Promise.all( views @@ -66,7 +104,12 @@ async function conditions(now: number) { .map(async (container) => [container.id, await docker('GET', `/containers/${container.id}/json`)] as const), ), ); - return [...hostConditions(host), ...containerConditions(views, inspects, now), ...(await backupConditions(now))]; + return [ + ...hostConditions(host, rules), + ...containerConditions(views, inspects, now, rules), + ...usageConditions(frame, rules), + ...(rules['backup-late'].enabled ? await backupConditions(now, undefined, rules['backup-late'].threshold) : []), + ]; } async function github(method: string, route: string, body: unknown) { @@ -86,7 +129,7 @@ async function github(method: string, route: string, body: unknown) { return (await response.json()) as { number: number }; } -async function notify(alerts: Alerts, resolved: Alert[]) { +async function issues(alerts: Alerts, resolved: Alert[]) { for (const alert of alerts.active().filter((candidate) => candidate.issue === null)) { const issue = await github('POST', '/issues', { title: `Alert: ${alert.title}`, @@ -101,13 +144,47 @@ async function notify(alerts: Alerts, resolved: Alert[]) { } } -export function monitor(alerts: Alerts, every = 60_000) { +export const emailing = () => Boolean(mailer && config.alertsEmail); + +export async function email(subject: string, text: string) { + if (!mailer || !config.alertsEmail) { + throw new Refusal(503, 'Email needs SMTP_URL and ALERTS_EMAIL in the console env file.'); + } + await mailer.sendMail({ from: config.alertsFrom ?? config.alertsEmail, to: config.alertsEmail, subject, text }); +} + +async function emails(opened: Alert[], resolved: Alert[]) { + for (const alert of opened) { + await email(`Alert: ${alert.title}`, `${alert.detail}\n\nSince ${new Date(alert.since).toISOString()}.\n${config.appUrl}/#/alerts`); + } + for (const alert of resolved) { + await email(`Resolved: ${alert.title}`, `Resolved at ${new Date(alert.resolved!).toISOString()} after starting ${new Date(alert.since).toISOString()}.\n${config.appUrl}/#/alerts`); + } +} + +export function monitor(alerts: Alerts, every = 10_000) { const tick = async () => { - const { resolved } = alerts.update(await conditions(Date.now())); - if (config.alertsToken) { - await notify(alerts, resolved); + const now = Date.now(); + const { opened, resolved } = alerts.update(await conditions(alerts.rules(), now), now); + const sent = await Promise.allSettled([ + ...(config.alertsToken ? [issues(alerts, resolved)] : []), + ...(emailing() && (opened.length || resolved.length) ? [emails(opened, resolved)] : []), + ]); + for (const result of sent) { + if (result.status === 'rejected') { + console.error('alert notification failed:', result.reason); + } } }; - const timer = setInterval(() => void tick().catch((error: unknown) => console.error('alerts failed:', error)), every); + let running = false; + const timer = setInterval(() => { + if (running) { + return; + } + running = true; + void tick() + .catch((error: unknown) => console.error('alerts failed:', error)) + .finally(() => (running = false)); + }, every); return () => clearInterval(timer); } diff --git a/app/src/server/terminal.ts b/app/src/server/terminal.ts index 4d1187f..d2bb554 100644 --- a/app/src/server/terminal.ts +++ b/app/src/server/terminal.ts @@ -1,7 +1,7 @@ import type http from 'node:http'; import type { WebSocket } from 'ws'; import { config } from './config.ts'; -import { Demuxer, docker, LogLines, stream, upgrade } from './docker.ts'; +import { Demuxer, docker, LogLines, stream, upgrade, type LogLine } from './docker.ts'; const shell = ['/bin/sh', '-c', 'if command -v bash >/dev/null; then exec bash; else exec sh; fi']; const hostShell = ['nsenter', '--target', '1', '--all', '--', '/bin/bash', '-l']; @@ -88,6 +88,27 @@ export async function logs(socket: WebSocket, container: string, params: URLSear }); } +export async function searchLogs(containers: { id: string; name: string }[], text: string, since: number, limit = 500) { + const needle = text.toLowerCase(); + const found: (LogLine & { container: string })[] = []; + await Promise.all( + containers.map(async (container) => { + const { output, read } = await readLogs(container.id, `follow=0&tail=20000&since=${Math.floor(since / 1000)}`); + let matches = 0; + for await (const chunk of output) { + for (const line of read(chunk as Buffer)) { + if (matches < limit && line.text.toLowerCase().includes(needle)) { + found.push({ ...line, container: container.name }); + matches += 1; + } + } + } + }), + ); + found.sort((a, b) => b.time.localeCompare(a.time)); + return { lines: found.slice(0, limit), more: found.length > limit }; +} + export async function downloadLogs(response: http.ServerResponse, container: { id: string; name: string }) { const { output, read } = await readLogs(container.id, 'tail=all'); response.writeHead(200, { diff --git a/app/src/web/App.tsx b/app/src/web/App.tsx index dfb5bce..49f3ad3 100644 --- a/app/src/web/App.tsx +++ b/app/src/web/App.tsx @@ -11,6 +11,7 @@ import { ContainerPage } from './pages/ContainerPage'; import { Docs } from './pages/Docs'; import { HostShell } from './pages/HostShell'; import { Images } from './pages/Images'; +import { LogSearch } from './pages/LogSearch'; import { Overview } from './pages/Overview'; import { ProjectPage } from './pages/ProjectPage'; import { SecretPage, Secrets } from './pages/Secrets'; @@ -48,7 +49,8 @@ function Page({ hash, user }: { hash: string; user: User }) { if (user.owner && hash === '/backups') return ; if (hash === '/images') return ; if (hash === '/volumes') return ; - if (hash === '/alerts') return ; + if (hash === '/alerts') return ; + if (hash === '/logs') return ; if (can(user, 'area:shell') && hash === '/shell') return ; if (user.owner && hash === '/admin') return ; return ; @@ -69,8 +71,9 @@ export function App() { }, []); const pages: ShellTab[] = [ - { href: '#/', label: 'Overview', icon: 'home', active: !/^\/(images|volumes|alerts|shell|admin|docs|secrets|backups)/.test(hash) }, + { href: '#/', label: 'Overview', icon: 'home', active: !/^\/(images|volumes|alerts|logs|shell|admin|docs|secrets|backups)/.test(hash) }, { href: '#/alerts', label: alerts ? `Alerts · ${alerts}` : 'Alerts', icon: 'bell', active: hash === '/alerts' }, + { href: '#/logs', label: 'Logs', icon: 'search', active: hash === '/logs' }, { href: '#/images', label: 'Images', icon: 'library', active: hash === '/images' }, { href: '#/volumes', label: 'Volumes', icon: 'folder', active: hash === '/volumes' }, ...(user && can(user, 'area:shell') ? ([{ href: '#/shell', label: 'Shell', icon: 'monitor', active: hash === '/shell' }] satisfies ShellTab[]) : []), diff --git a/app/src/web/api.ts b/app/src/web/api.ts index 0e894f6..667e69b 100644 --- a/app/src/web/api.ts +++ b/app/src/web/api.ts @@ -137,7 +137,38 @@ export interface SecretDetail extends Secret { export interface Point { at: number; cpu: number; + cpuPeak: number; memory: number; + memoryPeak: number; + net: number; + io: number; +} + +export type Metric = Exclude; + +export interface Traffic { + rx: number; + tx: number; + read: number; + write: number; +} + +export interface ContainerLive extends Traffic { + project: string; + cpu: number; + cpuLimit: number | null; + memory: number; + memoryLimit: number | null; +} + +export interface Frame { + at: number; + host: Host & Traffic; + containers: Record; +} + +export function rate(value: number | null | undefined) { + return value === null || value === undefined ? '—' : `${bytes(value)}/s`; } export interface Series { @@ -180,11 +211,29 @@ export interface Alert { issue: number | null; } +export interface AlertRule { + kind: string; + label: string; + unit: string | null; + threshold: number; + minutes: number; + enabled: boolean; +} + export interface AlertsData { active: Alert[]; recent: Alert[]; notifying: boolean; + emailing: boolean; repo: string; + rules: AlertRule[]; +} + +export interface LogMatch { + container: string; + stream: 'stdout' | 'stderr'; + time: string; + text: string; } export interface DocPage { diff --git a/app/src/web/components/Chart.tsx b/app/src/web/components/Chart.tsx index a2a7161..e18efba 100644 --- a/app/src/web/components/Chart.tsx +++ b/app/src/web/components/Chart.tsx @@ -1,5 +1,5 @@ import { useEffect, useRef, useState, type PointerEvent } from 'react'; -import type { Series } from '../api'; +import type { Metric, Series } from '../api'; const palette = ['#6a9eea', '#5fbf7f', '#e0915a', '#a388e8', '#4fb8b0', '#e07fb5', '#d7b24a', '#e5736b', '#8b8b95']; const height = 180; @@ -8,7 +8,7 @@ const pad = { top: 10, right: 10, bottom: 22, left: 58 }; export interface ChartProps { label: string; series: Series[]; - metric: 'cpu' | 'memory'; + metric: Metric; span: number; step: number; now: number; @@ -30,6 +30,7 @@ function timeLabel(at: number, span: number, precise = false) { return new Intl.DateTimeFormat(undefined, { ...(span > day || precise ? { day: 'numeric', month: 'short' } : {}), ...(span <= 7 * day || precise ? { hour: '2-digit', minute: '2-digit' } : {}), + ...(span <= 600_000 ? { second: '2-digit' } : {}), }).format(at); } @@ -44,7 +45,7 @@ function useWidth() { return { ref, width }; } -function Breakdown({ lines, at, metric, total, format }: { lines: Series[]; at: number; metric: 'cpu' | 'memory'; total: number; format: (value: number) => string }) { +function Breakdown({ lines, at, metric, total, format }: { lines: Series[]; at: number; metric: Metric; total: number; format: (value: number) => string }) { const parts = lines .map((line) => ({ name: line.name, value: line.points.find((point) => point.at === at)?.[metric] ?? 0 })) .filter((part) => part.value > 0) diff --git a/app/src/web/components/Metrics.tsx b/app/src/web/components/Metrics.tsx index f0fb4d5..73c6e82 100644 --- a/app/src/web/components/Metrics.tsx +++ b/app/src/web/components/Metrics.tsx @@ -1,36 +1,59 @@ import { useState } from 'react'; -import { bytes, percent, ranges, type HistoryData, type Range } from '../api'; +import { bytes, percent, rate, ranges, type HistoryData, type Metric, type Series } from '../api'; +import { containerPoint, hostPoint, liveSeconds, useLive } from '../live'; import { load, save } from '../stored'; import { usePoll } from '../usePoll'; import { Chart } from './Chart'; import { Segmented } from './Segmented'; -export function Metrics({ query, cpuMax, memoryMax }: { query: string; cpuMax?: number; memoryMax?: number }) { - const [range, setRange] = useState(() => { +const options = ['live', ...ranges] as const; +type Option = (typeof options)[number]; + +export type LiveTargets = 'host' | { name: string; label: string }[]; + +export function Metrics({ query, live, cpuMax, memoryMax }: { query: string; live: LiveTargets; cpuMax?: number; memoryMax?: number }) { + const [range, setRange] = useState
); } diff --git a/app/src/web/pages/ContainerPage.tsx b/app/src/web/pages/ContainerPage.tsx index 8e1f845..e037cc0 100644 --- a/app/src/web/pages/ContainerPage.tsx +++ b/app/src/web/pages/ContainerPage.tsx @@ -101,7 +101,7 @@ export function ContainerPage({ id, user }: { id: string; user: User }) { - +
diff --git a/app/src/web/pages/LogSearch.tsx b/app/src/web/pages/LogSearch.tsx new file mode 100644 index 0000000..604a6fa --- /dev/null +++ b/app/src/web/pages/LogSearch.tsx @@ -0,0 +1,73 @@ +import MuiTextField from '@mui/material/TextField'; +import { Button, PageHeader, Select } from '@project-graphite/ui'; +import { useState, type FormEvent } from 'react'; +import { get, type LogMatch, type Overview } from '../api'; +import { Segmented } from '../components/Segmented'; +import { usePoll } from '../usePoll'; + +const windows = ['15m', '1h', '6h', '24h', '7d'] as const; + +export function LogSearch() { + const projects = (usePoll('/api/overview', 60_000).data?.projects ?? []).map((project) => project.slug); + const [project, setProject] = useState(''); + const [since, setSince] = useState<(typeof windows)[number]>('1h'); + const [text, setText] = useState(''); + const [result, setResult] = useState<{ lines: LogMatch[]; more: boolean }>(); + const [message, setMessage] = useState(); + + const search = (event: FormEvent) => { + event.preventDefault(); + setMessage('Searching…'); + get<{ lines: LogMatch[]; more: boolean }>(`/api/logs/search?${new URLSearchParams({ q: text, since, ...(project ? { project } : {}) })}`).then( + (found) => { + setResult(found); + setMessage(undefined); + }, + (error: Error) => setMessage(error.message), + ); + }; + + return ( +
+ + Search every container's logs, newest first, case-insensitively. + +
+ setText(event.target.value)} size="small" type="search" value={text} /> +
+ ({ value: slug, label: slug }))]} - value={project} - /> -
- - - {message && {message}} - - {result && ( -
-

- {result.lines.length} lines{result.more ? ', the newest 500 shown' : ''} -

-
- - - {result.lines.map((line, index) => ( - - - - - - ))} - -
{line.time.slice(0, 19).replace('T', ' ')}{line.container}{line.text}
-
-
- )} -
- ); -} diff --git a/app/src/web/pages/Overview.tsx b/app/src/web/pages/Overview.tsx index 62e9eca..fda7cc8 100644 --- a/app/src/web/pages/Overview.tsx +++ b/app/src/web/pages/Overview.tsx @@ -3,13 +3,12 @@ import MuiLink from '@mui/material/Link'; import MuiTextField from '@mui/material/TextField'; import { EmptyState, ErrorState, Meter, PageHeader, TagChip, timeAgo } from '@project-graphite/ui'; import { useState } from 'react'; -import { bytes, completed, percent, rate, stampDate, type AlertsData, type Host, type Overview as OverviewData, type Project, type Traffic } from '../api'; +import { bytes, completed, percent, stampDate, type AlertsData, type Host, type Overview as OverviewData, type Project } from '../api'; import { Metrics } from '../components/Metrics'; -import { projectTotals, useLive } from '../live'; import { load, save } from '../stored'; import { usePoll } from '../usePoll'; -function HostCards({ host, traffic }: { host: Host; traffic?: Traffic }) { +function HostCards({ host }: { host: Host }) { const cards = [ { label: 'CPU', value: percent(host.cpu), detail: `${host.cpus} cores · load ${host.load.map((value) => value.toFixed(2)).join(' ')}`, ratio: host.cpu / 100 }, { label: 'Memory', value: bytes(host.memory.used), detail: `of ${bytes(host.memory.total)}`, ratio: host.memory.used / host.memory.total }, @@ -17,22 +16,15 @@ function HostCards({ host, traffic }: { host: Host; traffic?: Traffic }) { { label: 'Disk', value: bytes(host.disk.used), detail: `of ${bytes(host.disk.total)}`, ratio: host.disk.used / host.disk.total }, ]; return ( -
-
- {cards.map((card) => ( -
-

{card.label}

-

{card.value}

- -

{card.detail}

-
- ))} -
- {traffic && ( -

- network in {rate(traffic.rx)} · out {rate(traffic.tx)} · disk read {rate(traffic.read)} · written {rate(traffic.write)} · live -

- )} +
+ {cards.map((card) => ( +
+

{card.label}

+

{card.value}

+ +

{card.detail}

+
+ ))}
); } @@ -75,7 +67,7 @@ function status(project: Project) { return ; } -function ProjectList({ projects, live }: { projects: OverviewData['projects']; live: ReturnType }) { +function ProjectList({ projects }: { projects: OverviewData['projects'] }) { const [filter, setFilter] = useState(''); const [pinned, setPinned] = useState(() => load('console.pinned', [])); const toggle = (slug: string) => { @@ -129,8 +121,8 @@ function ProjectList({ projects, live }: { projects: OverviewData['projects']; l {project.needsRedeploy && } - {percent(live.get(project.slug)?.cpu ?? project.cpu)} - {bytes(live.get(project.slug)?.memory ?? project.memory)} + {percent(project.cpu)} + {bytes(project.memory)} {project.storage ? bytes(project.storage.volumes + project.storage.images) : '—'} ))} @@ -144,9 +136,7 @@ function ProjectList({ projects, live }: { projects: OverviewData['projects']; l export function Overview() { const { data, error, reload } = usePoll('/api/overview'); - const active = usePoll('/api/alerts', 30_000).data?.active ?? []; - const frame = useLive().at(-1); - const host = frame?.host ?? data?.host; + const active = usePoll('/api/alerts', 60_000).data?.active ?? []; if (error && !data) { return void reload()} title="The console could not read the VPS">{error.message}; } @@ -159,7 +149,7 @@ export function Overview() {

{active.length === 1 ? active[0].detail : active.map((alert) => alert.title).join(' · ')}

)} - {host && } + {data?.host && } {data && (

Backups:{' '} @@ -168,10 +158,10 @@ export function Overview() { : 'none has run yet'}

)} - {data?.host && } + {data?.host && } {data?.host && data.docker && } {data && data.projects.length === 0 && Nothing runs on the VPS yet.} - {data && data.projects.length > 0 && } + {data && data.projects.length > 0 && } ); } diff --git a/app/src/web/pages/ProjectPage.tsx b/app/src/web/pages/ProjectPage.tsx index 3a8ceba..8bfbb75 100644 --- a/app/src/web/pages/ProjectPage.tsx +++ b/app/src/web/pages/ProjectPage.tsx @@ -59,7 +59,7 @@ export function ProjectPage({ slug, user }: { slug: string; user: User }) { - ({ name: container.name, label: container.service }))} query={`project=${slug}`} /> +

Containers

diff --git a/app/test/alerts.spec.ts b/app/test/alerts.spec.ts index 6f06176..054dc80 100644 --- a/app/test/alerts.spec.ts +++ b/app/test/alerts.spec.ts @@ -2,11 +2,10 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; -import { Alerts, defaultRules } from '../src/server/alerts.ts'; +import { Alerts } from '../src/server/alerts.ts'; import { backupConditions } from '../src/server/backups.ts'; import type { ContainerView } from '../src/server/inventory.ts'; -import type { HostLive } from '../src/server/live.ts'; -import { containerConditions, hostConditions, usageConditions } from '../src/server/monitor.ts'; +import { containerConditions, hostConditions } from '../src/server/monitor.ts'; const directories: string[] = []; @@ -39,52 +38,33 @@ function container(service: string, overrides: Partial = {}): Con sleeps: false, cpu: 1, memory: 1, - memoryLimit: 100, + memoryLimit: 1, ...overrides, }; } -const condition = { key: 'disk', title: 'The VPS disk is 90% used', detail: 'Over 85%.', for: 120_000 }; +const condition = { key: 'disk', title: 'The VPS disk is 90% used', detail: 'Over 85%.' }; describe('Alerts', () => { - it('opens an alert once its condition has held for the rule duration, and resolves it after a clear minute', () => { + it('opens an alert only after two checks in a row, and resolves it after two clear checks', () => { const alerts = new Alerts(temporary()); - expect(alerts.update([condition], 0).opened).toEqual([]); - expect(alerts.update([condition], 110_000).opened).toEqual([]); - expect(alerts.update([condition], 120_000).opened.map((alert) => [alert.key, alert.since])).toEqual([['disk', 0]]); - expect(alerts.update([condition], 130_000).opened).toEqual([]); - expect(alerts.update([], 140_000).resolved).toEqual([]); - expect(alerts.update([], 190_000).resolved).toEqual([]); + expect(alerts.update([condition], 1).opened).toEqual([]); + expect(alerts.update([condition], 2).opened.map((alert) => alert.key)).toEqual(['disk']); + expect(alerts.update([condition], 3).opened).toEqual([]); + expect(alerts.update([], 4).resolved).toEqual([]); expect(alerts.active()).toHaveLength(1); - expect(alerts.update([], 200_000).resolved.map((alert) => alert.resolved)).toEqual([200_000]); + expect(alerts.update([], 5).resolved.map((alert) => alert.resolved)).toEqual([5]); expect(alerts.active()).toEqual([]); - expect(alerts.recent(210_000).map((alert) => alert.key)).toEqual(['disk']); + expect(alerts.recent(6).map((alert) => alert.key)).toEqual(['disk']); alerts.close(); }); - it('ignores a condition that clears before its duration, and keeps an alert that returns within the clear minute', () => { + it('ignores a condition that clears before it is confirmed', () => { const alerts = new Alerts(temporary()); - alerts.update([condition], 0); - alerts.update([], 60_000); - expect(alerts.update([condition], 130_000).opened).toEqual([]); - expect(alerts.update([condition], 250_000).opened).toHaveLength(1); - alerts.update([], 260_000); - alerts.update([condition], 290_000); - expect(alerts.update([], 330_000).resolved).toEqual([]); - expect(alerts.active()).toHaveLength(1); - alerts.close(); - }); - - it('keeps edited rules and falls back to the defaults', () => { - const directory = temporary(); - const alerts = new Alerts(directory); - expect(alerts.rules()['host-disk']).toEqual(defaultRules['host-disk']); - alerts.setRule({ kind: 'host-disk', threshold: 70, minutes: 0.5, enabled: false }); + alerts.update([condition], 1); + alerts.update([], 2); + expect(alerts.update([condition], 3).opened).toEqual([]); alerts.close(); - const reopened = new Alerts(directory); - expect(reopened.rules()['host-disk']).toEqual({ kind: 'host-disk', threshold: 70, minutes: 0.5, enabled: false }); - expect(reopened.rules()['host-cpu']).toEqual(defaultRules['host-cpu']); - reopened.close(); }); }); @@ -96,29 +76,6 @@ describe('rules', () => { expect(hostConditions(host).map((found) => found.key)).toEqual(['memory', 'disk']); }); - it('follows edited thresholds, durations and switches', () => { - const host = { cpu: 95, cpus: 2, memory: usage(91), swap: usage(50), disk: usage(86) }; - const rules = { ...defaultRules, 'host-cpu': { ...defaultRules['host-cpu'], minutes: 1 }, 'host-memory': { ...defaultRules['host-memory'], enabled: false }, 'host-swap': { ...defaultRules['host-swap'], threshold: 40 } }; - expect(hostConditions(host, rules).map((found) => [found.key, found.for])).toEqual([ - ['cpu', 60_000], - ['swap', 120_000], - ['disk', 120_000], - ]); - }); - - it("measures a container's CPU and memory against its own limits", () => { - const live = (cpu: number, cpuLimit: number | null, memory: number, memoryLimit: number | null) => ({ project: 'strata', cpu, cpuLimit, memory, memoryLimit, rx: 0, tx: 0, read: 0, write: 0 }); - const frame = { - host: { cpus: 4 } as HostLive, - containers: { - 'strata-app-1': live(70, 0.75, 95, 100), - 'strata-postgres-1': live(380, null, 10, null), - 'strata-redis-1': live(10, 0.25, 50, 100), - }, - }; - expect(usageConditions(frame).map((found) => found.key)).toEqual(['cpu:strata-app-1', 'memory:strata-app-1', 'cpu:strata-postgres-1']); - }); - it('flags stopped and unhealthy containers, but not finished jobs or sleeping projects', () => { const views = [ container('app', { health: 'unhealthy', status: 'Up 1 hour (unhealthy)' }), diff --git a/app/test/history.spec.ts b/app/test/history.spec.ts index 47f84b3..dc72a84 100644 --- a/app/test/history.spec.ts +++ b/app/test/history.spec.ts @@ -1,7 +1,6 @@ import { mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; -import { DatabaseSync } from 'node:sqlite'; import { afterEach, describe, expect, it } from 'vitest'; import { History, keep, projectTotals, ranges } from '../src/server/history.ts'; @@ -14,68 +13,49 @@ afterEach(() => { } }); -function folder() { +function open() { const directory = mkdtempSync(path.join(tmpdir(), 'console-history-')); directories.push(directory); - return directory; + return new History(directory); } -const sample = (target: string, cpu: number, memory: number, extra: { cpuPeak?: number; memoryPeak?: number; net?: number; io?: number } = {}) => ({ - target, - cpu, - cpuPeak: extra.cpuPeak ?? cpu, - memory, - memoryPeak: extra.memoryPeak ?? memory, - net: extra.net ?? 0, - io: extra.io ?? 0, -}); - describe('History', () => { const now = 1000 * ranges['30d'].step; it('keeps 30 days of samples per target, oldest first', () => { - const history = new History(folder()); - history.record(now - keep - 1, [sample('host', 1, 1)]); - history.record(now - 2 * minute, [sample('host', 2, 2)]); - history.record(now - minute, [sample('host', 4, 4), sample('strata-app-1', 3, 3)]); - history.record(now, [sample('host', 6, 6)]); + const history = open(); + history.record(now - keep - 1, [{ target: 'host', cpu: 1, memory: 1 }]); + history.record(now - 2 * minute, [{ target: 'host', cpu: 2, memory: 2 }]); + history.record(now - minute, [ + { target: 'host', cpu: 4, memory: 4 }, + { target: 'strata-app-1', cpu: 3, memory: 3 }, + ]); + history.record(now, [{ target: 'host', cpu: 6, memory: 6 }]); expect(history.series('host', '1h', now).map((point) => point.cpu)).toEqual([2, 4, 6]); expect(history.series('strata-app-1', '1h', now)).toHaveLength(1); expect(history.series('host', '30d', now - 2).map((point) => point.cpu)).not.toContain(1); history.close(); }); - it('averages samples into the range step and keeps the highest peak', () => { - const history = new History(folder()); - history.record(now - 2 * minute, [sample('host', 2, 20, { cpuPeak: 9, memoryPeak: 25, net: 100, io: 10 })]); - history.record(now - minute, [sample('host', 4, 40, { cpuPeak: 5, memoryPeak: 70, net: 300, io: 30 })]); - history.record(now, [sample('host', 6, 60)]); + it('averages samples into the range step', () => { + const history = open(); + history.record(now - 2 * minute, [{ target: 'host', cpu: 2, memory: 20 }]); + history.record(now - minute, [{ target: 'host', cpu: 4, memory: 40 }]); + history.record(now, [{ target: 'host', cpu: 6, memory: 60 }]); expect(history.series('host', '30d', now)).toEqual([ - { at: now - ranges['30d'].step, cpu: 3, cpuPeak: 9, memory: 30, memoryPeak: 70, net: 200, io: 20 }, - { at: now, cpu: 6, cpuPeak: 6, memory: 60, memoryPeak: 60, net: 0, io: 0 }, + { at: now - ranges['30d'].step, cpu: 3, memory: 30 }, + { at: now, cpu: 6, memory: 60 }, ]); history.close(); }); - - it('reads samples written before peaks and traffic were kept, using the average as the peak', () => { - const directory = folder(); - const legacy = new DatabaseSync(path.join(directory, 'metrics.db')); - legacy.exec('CREATE TABLE samples (at INTEGER NOT NULL, target TEXT NOT NULL, cpu REAL NOT NULL, memory INTEGER NOT NULL)'); - legacy.prepare('INSERT INTO samples VALUES (?, ?, ?, ?)').run(now, 'host', 7, 70); - legacy.close(); - const history = new History(directory); - expect(history.series('host', '1h', now)).toEqual([{ at: now, cpu: 7, cpuPeak: 7, memory: 70, memoryPeak: 70, net: 0, io: 0 }]); - history.close(); - }); }); describe('projectTotals', () => { it("adds up each project's containers per point, in whole-machine CPU", () => { - const point = (at: number, cpu: number, memory: number) => ({ at, cpu, cpuPeak: cpu, memory, memoryPeak: memory, net: 1, io: 2 }); - const series: Record[]> = { - 'strata-app-1': [point(1, 40, 300), point(2, 20, 310)], - 'strata-postgres-1': [point(1, 10, 100)], - 'console-app-1': [point(2, 4, 80)], + const series: Record = { + 'strata-app-1': [{ at: 1, cpu: 40, memory: 300 }, { at: 2, cpu: 20, memory: 310 }], + 'strata-postgres-1': [{ at: 1, cpu: 10, memory: 100 }], + 'console-app-1': [{ at: 2, cpu: 4, memory: 80 }], 'graphite-message-lab-web-1': [], }; const totals = projectTotals( @@ -89,14 +69,8 @@ describe('projectTotals', () => { 2, ); expect(totals).toEqual([ - { - name: 'strata', - points: [ - { at: 1, cpu: 25, cpuPeak: 25, memory: 400, memoryPeak: 400, net: 2, io: 4 }, - { at: 2, cpu: 10, cpuPeak: 10, memory: 310, memoryPeak: 310, net: 1, io: 2 }, - ], - }, - { name: 'console', points: [{ at: 2, cpu: 2, cpuPeak: 2, memory: 80, memoryPeak: 80, net: 1, io: 2 }] }, + { name: 'strata', points: [{ at: 1, cpu: 25, memory: 400 }, { at: 2, cpu: 10, memory: 310 }] }, + { name: 'console', points: [{ at: 2, cpu: 2, memory: 80 }] }, ]); }); }); diff --git a/app/test/host.spec.ts b/app/test/host.spec.ts index 1a1b338..d4a233f 100644 --- a/app/test/host.spec.ts +++ b/app/test/host.spec.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { cpuTimes, diskBytes, meminfo, networkBytes } from '../src/server/host.ts'; +import { cpuTimes, meminfo } from '../src/server/host.ts'; describe('host', () => { it('reads total and idle time from /proc/stat', () => { @@ -13,25 +13,4 @@ describe('host', () => { swap: { used: 500 * 1024, total: 2000 * 1024 }, }); }); - - it('counts traffic on physical interfaces only, from /proc/net/dev', () => { - const text = [ - 'Inter-| Receive | Transmit', - ' face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed', - ' lo: 9000 10 0 0 0 0 0 0 9000 10 0 0 0 0 0 0', - ' eth0: 1000 10 0 0 0 0 0 0 2000 20 0 0 0 0 0 0', - 'docker0: 500 5 0 0 0 0 0 0 600 6 0 0 0 0 0 0', - 'veth12ab: 70 1 0 0 0 0 0 0 80 1 0 0 0 0 0 0', - 'br-1f2e: 70 1 0 0 0 0 0 0 80 1 0 0 0 0 0 0', - ' ens4: 300 3 0 0 0 0 0 0 400 4 0 0 0 0 0 0', - ].join('\n'); - expect(networkBytes(text)).toEqual({ rx: 1300, tx: 2400 }); - }); - - it('counts sectors read and written on whole disks, from /proc/diskstats', () => { - const text = [' 8 0 sda 10 0 100 0 20 0 200 0 0 0 0', ' 8 1 sda1 10 0 100 0 20 0 200 0 0 0 0', ' 252 0 vda 1 0 4 0 2 0 8 0 0 0 0', ' 7 0 loop0 1 0 50 0 0 0 0 0 0 0 0'].join( - '\n', - ); - expect(diskBytes(text)).toEqual({ read: 104 * 512, write: 208 * 512 }); - }); }); diff --git a/app/test/live.spec.ts b/app/test/live.spec.ts deleted file mode 100644 index b728beb..0000000 --- a/app/test/live.spec.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { Sample } from '../src/server/history.ts'; -import { accumulate, rate, type Frame } from '../src/server/live.ts'; - -const traffic = (rx: number, tx: number, read: number, write: number) => ({ rx, tx, read, write }); - -function frame(at: number, hostCpu: number, hostMemory: number, appCpu: number, appMemory: number): Frame { - return { - at, - host: { cpu: hostCpu, cpus: 2, load: [0, 0, 0], uptime: 1, memory: { used: hostMemory, total: 100 }, swap: { used: 0, total: 0 }, disk: { used: 0, total: 0 }, ...traffic(10, 20, 30, 40) }, - containers: { 'strata-app-1': { project: 'strata', cpu: appCpu, cpuLimit: 0.75, memory: appMemory, memoryLimit: 512, ...traffic(1, 2, 3, 4) } }, - }; -} - -describe('rate', () => { - it('turns two counter readings into a per-second rate, never negative', () => { - expect(rate(3000, 1000, 2)).toBe(1000); - expect(rate(500, 1000, 2)).toBe(0); - expect(rate(3000, undefined, 2)).toBe(0); - expect(rate(3000, 1000, 0)).toBe(0); - }); -}); - -describe('accumulate', () => { - it('records each minute as the average and peak of its seconds', () => { - const recorded: { at: number; samples: Sample[] }[] = []; - const history = { record: (at: number, samples: Sample[]) => recorded.push({ at, samples }) }; - accumulate(history, frame(60_000, 10, 40, 50, 200)); - accumulate(history, frame(61_000, 30, 60, 70, 100)); - expect(recorded).toEqual([]); - accumulate(history, frame(120_000, 5, 5, 5, 5)); - expect(recorded).toEqual([ - { - at: 60_000, - samples: [ - { target: 'host', cpu: 20, cpuPeak: 30, memory: 50, memoryPeak: 60, net: 30, io: 70 }, - { target: 'strata-app-1', cpu: 60, cpuPeak: 70, memory: 150, memoryPeak: 200, net: 3, io: 7 }, - ], - }, - ]); - }); -});