From aa63347de575af244bd15357a93957431fadf94d Mon Sep 17 00:00:00 2001 From: Amr Mohammed El-Sheraey <141947355+AmrMsCLL@users.noreply.github.com> Date: Thu, 5 Nov 2026 04:58:24 +0200 Subject: [PATCH] fix: redeploy through a host unit, an honest image prune, accurate disk use and stricter requests Redeploy runs graphite-up as a transient systemd unit through nsenter instead of calling the agent, returns at once, and the page polls the unit's state and log, so Cloudflare's 100 second limit and a redeploy of the console itself both work. The missing-secret hint ignores names the project's values file supplies, and the needs-redeploy hint counts graphite-up's .applied marker as a rollout. Removing images now removes every unused image and refreshes disk use. Disk use is reported as df does. Every response carries the security headers, and a write needs this console's Origin or Sec-Fetch-Site same-origin. --- app/src/server/config.ts | 1 - app/src/server/host.ts | 2 +- app/src/server/main.ts | 73 +++++++++++++++--------- app/src/server/rollout.ts | 88 +++++++++++++++++++++++++++++ app/src/server/secrets.ts | 11 ---- app/src/web/components/Redeploy.tsx | 61 ++++++++++++++++++++ app/src/web/pages/Images.tsx | 15 +++-- app/src/web/pages/ProjectPage.tsx | 10 +++- app/src/web/pages/Secrets.tsx | 23 +------- 9 files changed, 212 insertions(+), 72 deletions(-) create mode 100644 app/src/server/rollout.ts create mode 100644 app/src/web/components/Redeploy.tsx diff --git a/app/src/server/config.ts b/app/src/server/config.ts index df29d3d..855a955 100644 --- a/app/src/server/config.ts +++ b/app/src/server/config.ts @@ -29,6 +29,5 @@ export const config = { sshTarget: process.env.SSH_TARGET ?? 'ubuntu@localhost', secretsKey: process.env.SECRETS_KEY || undefined, graphiteHome: process.env.GRAPHITE_HOME ?? '/srv/graphite', - agentUrl: process.env.AGENT_URL ?? 'http://172.17.0.1:9000', webRoot: new URL('../web/', import.meta.url), }; diff --git a/app/src/server/host.ts b/app/src/server/host.ts index 13d87f0..dc2e951 100644 --- a/app/src/server/host.ts +++ b/app/src/server/host.ts @@ -39,7 +39,7 @@ export async function hostStats() { cpus: os.cpus().length, load: load.split(' ').slice(0, 3).map(Number), uptime: Number(uptime.split(' ')[0]), - disk: { used: (disk.blocks - disk.bfree) * disk.bsize, total: disk.blocks * disk.bsize }, + disk: { used: (disk.blocks - disk.bfree) * disk.bsize, total: (disk.blocks - disk.bfree + disk.bavail) * disk.bsize }, ...meminfo(memory), }; } diff --git a/app/src/server/main.ts b/app/src/server/main.ts index e3b2a50..fea3cef 100644 --- a/app/src/server/main.ts +++ b/app/src/server/main.ts @@ -10,11 +10,12 @@ import { config } from './config.ts'; import { docker } from './docker.ts'; import { docs } from './docs.ts'; import { History, projectTotals, ranges, type Range } from './history.ts'; -import { collect, containers, diskUsage, projects, refresh, snapshot, type ContainerView } from './inventory.ts'; +import { collect, containers, diskUsage, projects, refresh, refreshDisk, snapshot, type ContainerView } from './inventory.ts'; import { monitor } from './monitor.ts'; import { open, openSocket, parseOpen } from './proxy.ts'; import { Refusal } from './refusal.ts'; -import { appliedCompose, composeReferences, parseKeys, projectPattern, SecretStore } from './secrets.ts'; +import { appliedAt, appliedCompose, appliedValueNames, redeployStatus, startRedeploy } from './rollout.ts'; +import { composeReferences, parseKeys, projectPattern, SecretStore } from './secrets.ts'; import { downloadLogs, logs, terminal } from './terminal.ts'; const history = new History(config.dataDir); @@ -143,18 +144,19 @@ function secretStore() { return store; } -function needsRedeploy(project: string, views: ContainerView[]) { +async function needsRedeploy(project: string, views: ContainerView[]) { const changed = store?.changedAt(project) ?? null; const created = views.filter((view) => view.project === project).map((view) => view.created); - return changed !== null && created.length > 0 && changed > Math.max(...created); + return changed !== null && created.length > 0 && changed > Math.max(...created, (await appliedAt(project)) ?? 0); } async function secretProject(project: string, views: ContainerView[]) { + const [compose, values] = await Promise.all([appliedCompose(project), appliedValueNames(project)]); const stored = new Set(secretStore().list(project).map((secret) => secret.name)); return { project, - needsRedeploy: needsRedeploy(project, views), - missing: [...composeReferences(await appliedCompose(project))].filter(([name, reference]) => reference.required && !stored.has(name)).map(([name]) => name), + needsRedeploy: await needsRedeploy(project, views), + missing: [...composeReferences(compose)].filter(([name, reference]) => reference.required && !stored.has(name) && !values.has(name)).map(([name]) => name), }; } @@ -166,7 +168,7 @@ async function secretsApi(user: User, request: http.IncomingMessage, route: stri const slugs = new Set([...secrets.projects(), ...views.map((view) => view.project)].filter((slug) => projectPattern.test(slug) && slug !== 'console' && slug !== 'other')); return { projects: await Promise.all([...slugs].sort().map((slug) => secretProject(slug, views))), secrets: secrets.list() }; } - const match = /^(GET|POST) \/api\/secrets\/([a-z0-9-]+)(?:\/([A-Z_][A-Z0-9_]*|import|delete|redeploy))?(?:\/(reveal|restore))?$/.exec(route); + const match = /^(GET|POST) \/api\/secrets\/([a-z0-9-]+)(?:\/([A-Z_][A-Z0-9_]*|import|delete))?(?:\/(reveal|restore))?$/.exec(route); if (!match) { throw new Refusal(404, 'Not found.'); } @@ -205,21 +207,6 @@ async function secretsApi(user: User, request: http.IncomingMessage, route: stri secrets.remove(project, names, user.login); return { ok: true }; } - if (target === 'redeploy' && !action) { - await requireNow(request, user, `project:${project}`); - const response = await fetch(`${config.agentUrl}/reapply`, { - method: 'POST', - headers: { Authorization: `Bearer ${secrets.value('agent', 'CONSOLE_TOKEN')}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ name: project, by: user.login }), - signal: AbortSignal.timeout(660_000), - }); - if (!response.ok) { - throw new Refusal(502, `The agent answered ${response.status}: ${await response.text()}`); - } - secrets.audit(user.login, 'redeploy', project, null); - await refresh(); - return { ok: true }; - } if (!action) { secrets.update(project, target, await body(request), user.login); return { ok: true }; @@ -273,7 +260,7 @@ async function api(user: User, request: http.IncomingMessage, url: URL) { buildCache: usage.buildCache, volumes: usage.volumes.reduce((sum, volume) => sum + Math.max(0, volume.size), 0), }, - projects: list.map((project) => ({ ...project, needsRedeploy: needsRedeploy(project.slug, project.containers) })), + projects: await Promise.all(list.map(async (project) => ({ ...project, needsRedeploy: await needsRedeploy(project.slug, project.containers) }))), }; } const project = /^GET \/api\/projects\/([a-z0-9-]+)$/.exec(route); @@ -340,8 +327,12 @@ async function api(user: User, request: http.IncomingMessage, url: URL) { } if (route === 'POST /api/prune') { await requireNow(request, user, 'area:images'); - const result = await docker<{ SpaceReclaimed: number }>('POST', `/images/prune?filters=${encodeURIComponent('{"dangling":["true"]}')}`); - return { reclaimed: result.SpaceReclaimed }; + const result = await docker<{ SpaceReclaimed: number; ImagesDeleted: { Deleted?: string }[] | null }>( + 'POST', + `/images/prune?filters=${encodeURIComponent('{"dangling":["false"]}')}`, + ); + await refreshDisk(); + return { reclaimed: result.SpaceReclaimed, removed: (result.ImagesDeleted ?? []).filter((entry) => entry.Deleted).length }; } const action = /^POST \/api\/containers\/([0-9a-f]{12,64})\/(start|stop|restart)$/.exec(route); if (action) { @@ -351,6 +342,16 @@ async function api(user: User, request: http.IncomingMessage, url: URL) { await refresh(); return { ok: true }; } + const redeploy = /^(GET|POST) \/api\/projects\/([a-z0-9-]+)\/redeploy$/.exec(route); + if (redeploy) { + if (redeploy[1] === 'GET') { + return redeployStatus(redeploy[2]); + } + await requireNow(request, user, `project:${redeploy[2]}`); + await startRedeploy(redeploy[2]); + store?.audit(user.login, 'redeploy', redeploy[2], null); + return { ok: true }; + } const wake = /^POST \/api\/projects\/([a-z0-9-]+)\/wake$/.exec(route); if (wake) { await requireNow(request, user, `project:${wake[1]}`); @@ -375,8 +376,6 @@ async function serveFile(response: http.ServerResponse, pathname: string) { 'Content-Type': types[extension] ?? 'application/octet-stream', 'Cache-Control': name.startsWith('assets/') ? 'public, max-age=31536000, immutable' : 'no-cache', 'Content-Security-Policy': policy, - 'X-Content-Type-Options': 'nosniff', - 'Referrer-Policy': 'same-origin', }); response.end(file); } catch { @@ -384,8 +383,26 @@ async function serveFile(response: http.ServerResponse, pathname: string) { } } +const headers = { + 'X-Content-Type-Options': 'nosniff', + 'X-Frame-Options': 'DENY', + 'Referrer-Policy': 'same-origin', + 'Cross-Origin-Opener-Policy': 'same-origin', + 'Cross-Origin-Resource-Policy': 'same-origin', + 'Permissions-Policy': 'camera=(), microphone=(), geolocation=(), payment=(), usb=()', + ...(config.production ? { 'Strict-Transport-Security': 'max-age=63072000; includeSubDomains' } : {}), +}; + +function sameOrigin(request: http.IncomingMessage) { + const origin = request.headers.origin; + return origin ? origin === new URL(config.appUrl).origin : request.headers['sec-fetch-site'] === 'same-origin'; +} + async function handle(request: http.IncomingMessage, response: http.ServerResponse) { const url = new URL(request.url ?? '/', config.appUrl); + for (const [name, value] of Object.entries(headers)) { + response.setHeader(name, value); + } if (url.pathname === '/healthz') { response.writeHead(200).end('ok'); return; @@ -422,7 +439,7 @@ async function handle(request: http.IncomingMessage, response: http.ServerRespon return download(response, archive[1], archive[2], user.login); } if (url.pathname.startsWith('/api/')) { - if (request.method === 'POST' && request.headers.origin && request.headers.origin !== new URL(config.appUrl).origin) { + if (request.method !== 'GET' && !sameOrigin(request)) { throw new Refusal(403, 'Cross-origin requests are refused.'); } const importing = /^\/api\/backups\/([a-z0-9][a-z0-9_-]*)\/import$/.exec(url.pathname); diff --git a/app/src/server/rollout.ts b/app/src/server/rollout.ts new file mode 100644 index 0000000..d92d6d8 --- /dev/null +++ b/app/src/server/rollout.ts @@ -0,0 +1,88 @@ +import { readFile, stat } from 'node:fs/promises'; +import path from 'node:path'; +import { config } from './config.ts'; +import { Demuxer, docker, stream } from './docker.ts'; +import { Refusal } from './refusal.ts'; + +function applied(project: string, file: string) { + return path.join(config.hostRoot, config.graphiteHome, project, file); +} + +function missingIsEmpty(error: NodeJS.ErrnoException) { + if (error.code === 'ENOENT') { + return undefined; + } + throw error; +} + +export async function appliedCompose(project: string) { + return (await readFile(applied(project, 'compose.yaml'), 'utf8').catch(missingIsEmpty)) ?? ''; +} + +export async function appliedValueNames(project: string) { + const text = (await readFile(applied(project, 'values'), 'utf8').catch(missingIsEmpty)) ?? ''; + return new Set(text.split('\n').flatMap((line) => /^([A-Z_][A-Z0-9_]*)=/.exec(line)?.[1] ?? [])); +} + +export async function appliedAt(project: string) { + return (await stat(applied(project, '.applied')).catch(missingIsEmpty))?.mtimeMs ?? null; +} + +export async function onHost(command: string[]) { + const { Id } = await docker<{ Id: string }>('POST', `/containers/${config.self}/exec`, { + AttachStdout: true, + AttachStderr: true, + Tty: false, + Cmd: ['nsenter', '--target', '1', '--all', '--', ...command], + }); + const demuxer = new Demuxer(); + const chunks: Buffer[] = []; + for await (const chunk of await stream('POST', `/exec/${Id}/start`, { Detach: false, Tty: false })) { + chunks.push(...demuxer.push(chunk as Buffer).map((frame) => frame.data)); + } + const { ExitCode } = await docker<{ ExitCode: number }>('GET', `/exec/${Id}/json`); + return { code: ExitCode, output: Buffer.concat(chunks).toString('utf8').trim() }; +} + +const unit = (project: string) => `graphite-redeploy-${project}`; + +export async function redeployStatus(project: string) { + const { output } = await onHost(['systemctl', 'show', unit(project), '--property=ActiveState,SubState,InvocationID,StateChangeTimestamp']); + const properties = Object.fromEntries(output.split('\n').map((line) => [line.slice(0, line.indexOf('=')), line.slice(line.indexOf('=') + 1)])); + const state = + properties.ActiveState === 'activating' || properties.SubState === 'running' + ? 'running' + : properties.ActiveState === 'failed' + ? 'failed' + : properties.SubState === 'exited' + ? 'done' + : 'none'; + const log = + state !== 'none' && properties.InvocationID + ? (await onHost(['journalctl', `_SYSTEMD_INVOCATION_ID=${properties.InvocationID}`, '--no-pager', '--output=cat', '--lines=30'])).output + : ''; + return { state, log, at: properties.StateChangeTimestamp || null }; +} + +export async function startRedeploy(project: string) { + if ((await stat(applied(project, 'compose.yaml')).catch(missingIsEmpty)) === undefined) { + throw new Refusal(404, `${project} has not been deployed by the agent.`); + } + if ((await redeployStatus(project)).state === 'running') { + throw new Refusal(409, `${project} is already redeploying.`); + } + await onHost(['systemctl', 'reset-failed', unit(project)]); + await onHost(['systemctl', 'stop', unit(project)]); + const started = await onHost([ + 'systemd-run', + `--unit=${unit(project)}`, + `--description=Redeploy ${project} from the console`, + '--remain-after-exit', + '--property=RuntimeMaxSec=900', + path.posix.join(config.graphiteHome, 'bin', 'graphite-up'), + project, + ]); + if (started.code !== 0) { + throw new Refusal(502, `The host refused to start the redeploy: ${started.output}`); + } +} diff --git a/app/src/server/secrets.ts b/app/src/server/secrets.ts index e9f1e51..e3e3384 100644 --- a/app/src/server/secrets.ts +++ b/app/src/server/secrets.ts @@ -1,9 +1,7 @@ import { createCipheriv, createDecipheriv, createHash, randomBytes } from 'node:crypto'; import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, statSync, writeFileSync } from 'node:fs'; -import { readFile } from 'node:fs/promises'; import path from 'node:path'; import { DatabaseSync } from 'node:sqlite'; -import { config } from './config.ts'; import { Refusal } from './refusal.ts'; export const projectPattern = /^[a-z0-9](?:[a-z0-9-]{0,38}[a-z0-9])?$/; @@ -130,15 +128,6 @@ export function composeReferences(text: string) { return references; } -export async function appliedCompose(project: string) { - return readFile(path.join(config.hostRoot, config.graphiteHome, project, 'compose.yaml'), 'utf8').catch((error: NodeJS.ErrnoException) => { - if (error.code === 'ENOENT') { - return ''; - } - throw error; - }); -} - export class SecretStore { private readonly db: DatabaseSync; diff --git a/app/src/web/components/Redeploy.tsx b/app/src/web/components/Redeploy.tsx new file mode 100644 index 0000000..00d0403 --- /dev/null +++ b/app/src/web/components/Redeploy.tsx @@ -0,0 +1,61 @@ +import { Button } from '@project-graphite/ui'; +import { useEffect, useState } from 'react'; +import { get, post } from '../api'; + +interface Status { + state: 'none' | 'running' | 'done' | 'failed'; + log: string; +} + +export function Redeploy({ project, onDone }: { project: string; onDone: () => void }) { + const [watching, setWatching] = useState(false); + const [status, setStatus] = useState(); + const [message, setMessage] = useState(); + + useEffect(() => { + if (!watching) { + return; + } + const timer = setInterval(() => { + get(`/api/projects/${project}/redeploy`).then( + (next) => { + setStatus(next); + setMessage(undefined); + if (next.state === 'done' || next.state === 'failed') { + setWatching(false); + onDone(); + } + }, + () => setMessage('Waiting for the console to answer…'), + ); + }, 3000); + return () => clearInterval(timer); + }, [watching, project, onDone]); + + const start = () => { + setStatus(undefined); + setMessage('Starting…'); + post(`/api/projects/${project}/redeploy`).then( + () => { + setMessage(undefined); + setStatus({ state: 'running', log: '' }); + setWatching(true); + }, + (error: Error) => setMessage(error.message), + ); + }; + + return ( + + + + + {message ?? (status?.state === 'running' ? 'Redeploying…' : status?.state === 'done' ? 'Redeployed.' : status?.state === 'failed' ? 'The redeploy failed:' : '')} + + + {status?.state === 'failed' && status.log && 
{status.log}
} + + ); +} diff --git a/app/src/web/pages/Images.tsx b/app/src/web/pages/Images.tsx index 9a21ef5..23c2596 100644 --- a/app/src/web/pages/Images.tsx +++ b/app/src/web/pages/Images.tsx @@ -7,7 +7,7 @@ export function Images({ user }: { user: User }) { const images = usePoll('/api/images', 30_000); const disk = usePoll('/api/disk', 30_000); const [pruning, setPruning] = useState(false); - const [reclaimed, setReclaimed] = useState(); + const [result, setResult] = useState<{ reclaimed: number; removed: number }>(); if (images.error && !images.data) { return {images.error.message}; @@ -19,7 +19,7 @@ export function Images({ user }: { user: User }) { setPruning(true)}>Remove dangling images + ) } title="Images" @@ -27,7 +27,7 @@ export function Images({ user }: { user: User }) { {disk.data && ( images {bytes(disk.data.images)} · containers {bytes(disk.data.containers)} · build cache {bytes(disk.data.buildCache)} - {reclaimed !== undefined && ` · reclaimed ${bytes(reclaimed)}`} + {result && ` · removed ${result.removed} images, reclaimed ${bytes(result.reclaimed)}`} )} @@ -59,13 +59,12 @@ export function Images({ user }: { user: User }) { confirmLabel="Remove" onClose={() => setPruning(false)} onConfirm={async () => { - const result = await post<{ reclaimed: number }>('/api/prune'); - setReclaimed(result.reclaimed); - await images.reload(); + setResult(await post<{ reclaimed: number; removed: number }>('/api/prune')); + await Promise.all([images.reload(), disk.reload()]); }} - title="Remove dangling images?" + title={`Remove ${unused.length} unused images?`} > - Removes untagged image layers no container uses. Tagged images stay. + Removes every image no container uses, running or stopped, older builds included. A project pulls its image again when it next deploys. )} diff --git a/app/src/web/pages/ProjectPage.tsx b/app/src/web/pages/ProjectPage.tsx index a0ff15f..8bfbb75 100644 --- a/app/src/web/pages/ProjectPage.tsx +++ b/app/src/web/pages/ProjectPage.tsx @@ -3,6 +3,7 @@ import { Button, ErrorState, PageHeader, TagChip } from '@project-graphite/ui'; import { useState } from 'react'; import { bytes, can, percent, post, shortImage, stateColor, stateOf, type ProjectDetail, type User } from '../api'; import { Logs } from '../components/Logs'; +import { Redeploy } from '../components/Redeploy'; import { Metrics } from '../components/Metrics'; import { usePoll } from '../usePoll'; import { SecretList } from './Secrets'; @@ -39,7 +40,14 @@ export function ProjectPage({ slug, user }: { slug: string; user: User }) { return (
void reload()} slug={slug} />} + actions={ + can(user, `project:${slug}`) && ( + + void reload()} project={slug} /> + {project.sleeps && void reload()} slug={slug} />} + + ) + } eyebrow={Overview} title={slug} > diff --git a/app/src/web/pages/Secrets.tsx b/app/src/web/pages/Secrets.tsx index 2f47ee0..aa6bec4 100644 --- a/app/src/web/pages/Secrets.tsx +++ b/app/src/web/pages/Secrets.tsx @@ -18,6 +18,7 @@ import { } from '@project-graphite/ui'; import { useEffect, useState } from 'react'; import { can, post, type Secret, type SecretDetail, type SecretProject, type SecretType, type User } from '../api'; +import { Redeploy } from '../components/Redeploy'; import { Segmented } from '../components/Segmented'; import { usePoll } from '../usePoll'; @@ -28,28 +29,6 @@ const typeOptions: SelectOption[] = [ const when = (at: number) => timeAgo(new Date(at).toISOString()); -function Redeploy({ project, onDone }: { project: string; onDone: () => void }) { - const [message, setMessage] = useState(); - const redeploy = () => { - setMessage('Redeploying…'); - post(`/api/secrets/${project}/redeploy`).then( - () => { - setMessage('Redeployed.'); - onDone(); - }, - (error: Error) => setMessage(error.message), - ); - }; - return ( - - - {message && {message}} - - ); -} - function SecretSheet({ prefix, secret, onClose, onSaved }: { prefix: string; secret?: Secret; onClose: () => void; onSaved: () => void }) { const [mode, setMode] = useState<'Create' | 'Import'>('Create'); const [path, setPath] = useState(secret ? `/${secret.project}/${secret.name}` : prefix);