Revert 21 feat/live monitoring

app/src/server/config.ts

@@ -29,6 +29,5 @@ export const config = {
   sshTarget: process.env.SSH_TARGET ?? 'ubuntu@localhost',
   secretsKey: process.env.SECRETS_KEY || undefined,
   graphiteHome: process.env.GRAPHITE_HOME ?? '/srv/graphite',
-  agentUrl: process.env.AGENT_URL ?? 'http://172.17.0.1:9000',
   webRoot: new URL('../web/', import.meta.url),
 };

app/src/server/host.ts

@@ -39,7 +39,7 @@ export async function hostStats() {
     cpus: os.cpus().length,
     load: load.split(' ').slice(0, 3).map(Number),
     uptime: Number(uptime.split(' ')[0]),
-    disk: { used: (disk.blocks - disk.bfree) * disk.bsize, total: disk.blocks * disk.bsize },
+    disk: { used: (disk.blocks - disk.bfree) * disk.bsize, total: (disk.blocks - disk.bfree + disk.bavail) * disk.bsize },
     ...meminfo(memory),
   };
 }

app/src/server/main.ts

@@ -10,11 +10,12 @@ import { config } from './config.ts';
 import { docker } from './docker.ts';
 import { docs } from './docs.ts';
 import { History, projectTotals, ranges, type Range } from './history.ts';
-import { collect, containers, diskUsage, projects, refresh, snapshot, type ContainerView } from './inventory.ts';
+import { collect, containers, diskUsage, projects, refresh, refreshDisk, snapshot, type ContainerView } from './inventory.ts';
 import { monitor } from './monitor.ts';
 import { open, openSocket, parseOpen } from './proxy.ts';
 import { Refusal } from './refusal.ts';
-import { appliedCompose, composeReferences, parseKeys, projectPattern, SecretStore } from './secrets.ts';
+import { appliedAt, appliedCompose, appliedValueNames, redeployStatus, startRedeploy } from './rollout.ts';
+import { composeReferences, parseKeys, projectPattern, SecretStore } from './secrets.ts';
 import { downloadLogs, logs, terminal } from './terminal.ts';
 
 const history = new History(config.dataDir);
@@ -143,18 +144,19 @@ function secretStore() {
   return store;
 }
 
-function needsRedeploy(project: string, views: ContainerView[]) {
+async function needsRedeploy(project: string, views: ContainerView[]) {
   const changed = store?.changedAt(project) ?? null;
   const created = views.filter((view) => view.project === project).map((view) => view.created);
-  return changed !== null && created.length > 0 && changed > Math.max(...created);
+  return changed !== null && created.length > 0 && changed > Math.max(...created, (await appliedAt(project)) ?? 0);
 }
 
 async function secretProject(project: string, views: ContainerView[]) {
+  const [compose, values] = await Promise.all([appliedCompose(project), appliedValueNames(project)]);
   const stored = new Set(secretStore().list(project).map((secret) => secret.name));
   return {
     project,
-    needsRedeploy: needsRedeploy(project, views),
-    missing: [...composeReferences(await appliedCompose(project))].filter(([name, reference]) => reference.required && !stored.has(name)).map(([name]) => name),
+    needsRedeploy: await needsRedeploy(project, views),
+    missing: [...composeReferences(compose)].filter(([name, reference]) => reference.required && !stored.has(name) && !values.has(name)).map(([name]) => name),
   };
 }
 
@@ -166,7 +168,7 @@ async function secretsApi(user: User, request: http.IncomingMessage, route: stri
     const slugs = new Set([...secrets.projects(), ...views.map((view) => view.project)].filter((slug) => projectPattern.test(slug) && slug !== 'console' && slug !== 'other'));
     return { projects: await Promise.all([...slugs].sort().map((slug) => secretProject(slug, views))), secrets: secrets.list() };
   }
-  const match = /^(GET|POST) \/api\/secrets\/([a-z0-9-]+)(?:\/([A-Z_][A-Z0-9_]*|import|delete|redeploy))?(?:\/(reveal|restore))?$/.exec(route);
+  const match = /^(GET|POST) \/api\/secrets\/([a-z0-9-]+)(?:\/([A-Z_][A-Z0-9_]*|import|delete))?(?:\/(reveal|restore))?$/.exec(route);
   if (!match) {
     throw new Refusal(404, 'Not found.');
   }
@@ -205,21 +207,6 @@ async function secretsApi(user: User, request: http.IncomingMessage, route: stri
     secrets.remove(project, names, user.login);
     return { ok: true };
   }
-  if (target === 'redeploy' && !action) {
-    await requireNow(request, user, `project:${project}`);
-    const response = await fetch(`${config.agentUrl}/reapply`, {
-      method: 'POST',
-      headers: { Authorization: `Bearer ${secrets.value('agent', 'CONSOLE_TOKEN')}`, 'Content-Type': 'application/json' },
-      body: JSON.stringify({ name: project, by: user.login }),
-      signal: AbortSignal.timeout(660_000),
-    });
-    if (!response.ok) {
-      throw new Refusal(502, `The agent answered ${response.status}: ${await response.text()}`);
-    }
-    secrets.audit(user.login, 'redeploy', project, null);
-    await refresh();
-    return { ok: true };
-  }
   if (!action) {
     secrets.update(project, target, await body(request), user.login);
     return { ok: true };
@@ -273,7 +260,7 @@ async function api(user: User, request: http.IncomingMessage, url: URL) {
         buildCache: usage.buildCache,
         volumes: usage.volumes.reduce((sum, volume) => sum + Math.max(0, volume.size), 0),
       },
-      projects: list.map((project) => ({ ...project, needsRedeploy: needsRedeploy(project.slug, project.containers) })),
+      projects: await Promise.all(list.map(async (project) => ({ ...project, needsRedeploy: await needsRedeploy(project.slug, project.containers) }))),
     };
   }
   const project = /^GET \/api\/projects\/([a-z0-9-]+)$/.exec(route);
@@ -340,8 +327,12 @@ async function api(user: User, request: http.IncomingMessage, url: URL) {
   }
   if (route === 'POST /api/prune') {
     await requireNow(request, user, 'area:images');
-    const result = await docker<{ SpaceReclaimed: number }>('POST', `/images/prune?filters=${encodeURIComponent('{"dangling":["true"]}')}`);
-    return { reclaimed: result.SpaceReclaimed };
+    const result = await docker<{ SpaceReclaimed: number; ImagesDeleted: { Deleted?: string }[] | null }>(
+      'POST',
+      `/images/prune?filters=${encodeURIComponent('{"dangling":["false"]}')}`,
+    );
+    await refreshDisk();
+    return { reclaimed: result.SpaceReclaimed, removed: (result.ImagesDeleted ?? []).filter((entry) => entry.Deleted).length };
   }
   const action = /^POST \/api\/containers\/([0-9a-f]{12,64})\/(start|stop|restart)$/.exec(route);
   if (action) {
@@ -351,6 +342,16 @@ async function api(user: User, request: http.IncomingMessage, url: URL) {
     await refresh();
     return { ok: true };
   }
+  const redeploy = /^(GET|POST) \/api\/projects\/([a-z0-9-]+)\/redeploy$/.exec(route);
+  if (redeploy) {
+    if (redeploy[1] === 'GET') {
+      return redeployStatus(redeploy[2]);
+    }
+    await requireNow(request, user, `project:${redeploy[2]}`);
+    await startRedeploy(redeploy[2]);
+    store?.audit(user.login, 'redeploy', redeploy[2], null);
+    return { ok: true };
+  }
   const wake = /^POST \/api\/projects\/([a-z0-9-]+)\/wake$/.exec(route);
   if (wake) {
     await requireNow(request, user, `project:${wake[1]}`);
@@ -375,17 +376,33 @@ async function serveFile(response: http.ServerResponse, pathname: string) {
       'Content-Type': types[extension] ?? 'application/octet-stream',
       'Cache-Control': name.startsWith('assets/') ? 'public, max-age=31536000, immutable' : 'no-cache',
       'Content-Security-Policy': policy,
-      'X-Content-Type-Options': 'nosniff',
-      'Referrer-Policy': 'same-origin',
     });
     response.end(file);
   } catch {
     response.writeHead(404).end('Not found.');
   }
 }
 
+const headers = {
+  'X-Content-Type-Options': 'nosniff',
+  'X-Frame-Options': 'DENY',
+  'Referrer-Policy': 'same-origin',
+  'Cross-Origin-Opener-Policy': 'same-origin',
+  'Cross-Origin-Resource-Policy': 'same-origin',
+  'Permissions-Policy': 'camera=(), microphone=(), geolocation=(), payment=(), usb=()',
+  ...(config.production ? { 'Strict-Transport-Security': 'max-age=63072000; includeSubDomains' } : {}),
+};
+
+function sameOrigin(request: http.IncomingMessage) {
+  const origin = request.headers.origin;
+  return origin ? origin === new URL(config.appUrl).origin : request.headers['sec-fetch-site'] === 'same-origin';
+}
+
 async function handle(request: http.IncomingMessage, response: http.ServerResponse) {
   const url = new URL(request.url ?? '/', config.appUrl);
+  for (const [name, value] of Object.entries(headers)) {
+    response.setHeader(name, value);
+  }
   if (url.pathname === '/healthz') {
     response.writeHead(200).end('ok');
     return;
@@ -422,7 +439,7 @@ async function handle(request: http.IncomingMessage, response: http.ServerRespon
     return download(response, archive[1], archive[2], user.login);
   }
   if (url.pathname.startsWith('/api/')) {
-    if (request.method === 'POST' && request.headers.origin && request.headers.origin !== new URL(config.appUrl).origin) {
+    if (request.method !== 'GET' && !sameOrigin(request)) {
       throw new Refusal(403, 'Cross-origin requests are refused.');
     }
     const importing = /^\/api\/backups\/([a-z0-9][a-z0-9_-]*)\/import$/.exec(url.pathname);

app/src/server/rollout.ts

@@ -0,0 +1,88 @@
+import { readFile, stat } from 'node:fs/promises';
+import path from 'node:path';
+import { config } from './config.ts';
+import { Demuxer, docker, stream } from './docker.ts';
+import { Refusal } from './refusal.ts';
+
+function applied(project: string, file: string) {
+  return path.join(config.hostRoot, config.graphiteHome, project, file);
+}
+
+function missingIsEmpty(error: NodeJS.ErrnoException) {
+  if (error.code === 'ENOENT') {
+    return undefined;
+  }
+  throw error;
+}
+
+export async function appliedCompose(project: string) {
+  return (await readFile(applied(project, 'compose.yaml'), 'utf8').catch(missingIsEmpty)) ?? '';
+}
+
+export async function appliedValueNames(project: string) {
+  const text = (await readFile(applied(project, 'values'), 'utf8').catch(missingIsEmpty)) ?? '';
+  return new Set(text.split('\n').flatMap((line) => /^([A-Z_][A-Z0-9_]*)=/.exec(line)?.[1] ?? []));
+}
+
+export async function appliedAt(project: string) {
+  return (await stat(applied(project, '.applied')).catch(missingIsEmpty))?.mtimeMs ?? null;
+}
+
+export async function onHost(command: string[]) {
+  const { Id } = await docker<{ Id: string }>('POST', `/containers/${config.self}/exec`, {
+    AttachStdout: true,
+    AttachStderr: true,
+    Tty: false,
+    Cmd: ['nsenter', '--target', '1', '--all', '--', ...command],
+  });
+  const demuxer = new Demuxer();
+  const chunks: Buffer[] = [];
+  for await (const chunk of await stream('POST', `/exec/${Id}/start`, { Detach: false, Tty: false })) {
+    chunks.push(...demuxer.push(chunk as Buffer).map((frame) => frame.data));
+  }
+  const { ExitCode } = await docker<{ ExitCode: number }>('GET', `/exec/${Id}/json`);
+  return { code: ExitCode, output: Buffer.concat(chunks).toString('utf8').trim() };
+}
+
+const unit = (project: string) => `graphite-redeploy-${project}`;
+
+export async function redeployStatus(project: string) {
+  const { output } = await onHost(['systemctl', 'show', unit(project), '--property=ActiveState,SubState,InvocationID,StateChangeTimestamp']);
+  const properties = Object.fromEntries(output.split('\n').map((line) => [line.slice(0, line.indexOf('=')), line.slice(line.indexOf('=') + 1)]));
+  const state =
+    properties.ActiveState === 'activating' || properties.SubState === 'running'
+      ? 'running'
+      : properties.ActiveState === 'failed'
+        ? 'failed'
+        : properties.SubState === 'exited'
+          ? 'done'
+          : 'none';
+  const log =
+    state !== 'none' && properties.InvocationID
+      ? (await onHost(['journalctl', `_SYSTEMD_INVOCATION_ID=${properties.InvocationID}`, '--no-pager', '--output=cat', '--lines=30'])).output
+      : '';
+  return { state, log, at: properties.StateChangeTimestamp || null };
+}
+
+export async function startRedeploy(project: string) {
+  if ((await stat(applied(project, 'compose.yaml')).catch(missingIsEmpty)) === undefined) {
+    throw new Refusal(404, `${project} has not been deployed by the agent.`);
+  }
+  if ((await redeployStatus(project)).state === 'running') {
+    throw new Refusal(409, `${project} is already redeploying.`);
+  }
+  await onHost(['systemctl', 'reset-failed', unit(project)]);
+  await onHost(['systemctl', 'stop', unit(project)]);
+  const started = await onHost([
+    'systemd-run',
+    `--unit=${unit(project)}`,
+    `--description=Redeploy ${project} from the console`,
+    '--remain-after-exit',
+    '--property=RuntimeMaxSec=900',
+    path.posix.join(config.graphiteHome, 'bin', 'graphite-up'),
+    project,
+  ]);
+  if (started.code !== 0) {
+    throw new Refusal(502, `The host refused to start the redeploy: ${started.output}`);
+  }
+}

app/src/server/secrets.ts

@@ -1,9 +1,7 @@
 import { createCipheriv, createDecipheriv, createHash, randomBytes } from 'node:crypto';
 import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, statSync, writeFileSync } from 'node:fs';
-import { readFile } from 'node:fs/promises';
 import path from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
-import { config } from './config.ts';
 import { Refusal } from './refusal.ts';
 
 export const projectPattern = /^[a-z0-9](?:[a-z0-9-]{0,38}[a-z0-9])?$/;
@@ -130,15 +128,6 @@ export function composeReferences(text: string) {
   return references;
 }
 
-export async function appliedCompose(project: string) {
-  return readFile(path.join(config.hostRoot, config.graphiteHome, project, 'compose.yaml'), 'utf8').catch((error: NodeJS.ErrnoException) => {
-    if (error.code === 'ENOENT') {
-      return '';
-    }
-    throw error;
-  });
-}
-
 export class SecretStore {
   private readonly db: DatabaseSync;
 

app/src/web/components/Redeploy.tsx

@@ -0,0 +1,61 @@
+import { Button } from '@project-graphite/ui';
+import { useEffect, useState } from 'react';
+import { get, post } from '../api';
+
+interface Status {
+  state: 'none' | 'running' | 'done' | 'failed';
+  log: string;
+}
+
+export function Redeploy({ project, onDone }: { project: string; onDone: () => void }) {
+  const [watching, setWatching] = useState(false);
+  const [status, setStatus] = useState<Status>();
+  const [message, setMessage] = useState<string>();
+
+  useEffect(() => {
+    if (!watching) {
+      return;
+    }
+    const timer = setInterval(() => {
+      get<Status>(`/api/projects/${project}/redeploy`).then(
+        (next) => {
+          setStatus(next);
+          setMessage(undefined);
+          if (next.state === 'done' || next.state === 'failed') {
+            setWatching(false);
+            onDone();
+          }
+        },
+        () => setMessage('Waiting for the console to answer…'),
+      );
+    }, 3000);
+    return () => clearInterval(timer);
+  }, [watching, project, onDone]);
+
+  const start = () => {
+    setStatus(undefined);
+    setMessage('Starting…');
+    post(`/api/projects/${project}/redeploy`).then(
+      () => {
+        setMessage(undefined);
+        setStatus({ state: 'running', log: '' });
+        setWatching(true);
+      },
+      (error: Error) => setMessage(error.message),
+    );
+  };
+
+  return (
+    <span className="flex flex-col gap-1">
+      <span className="flex items-center gap-2">
+        <Button disabled={watching} onClick={start} size="small">
+          Redeploy
+        </Button>
+        <span className="mono-sm text-muted">
+          {message ?? (status?.state === 'running' ? 'Redeploying…' : status?.state === 'done' ? 'Redeployed.' : status?.state === 'failed' ? 'The redeploy failed:' : '')}
+        </span>
+      </span>
+      {status?.state === 'failed' && status.log && <pre className="mono-sm max-h-48 overflow-auto whitespace-pre-wrap text-faint">{status.log}</pre>}
+    </span>
+  );
+}

app/src/web/pages/Images.tsx

@@ -7,7 +7,7 @@ export function Images({ user }: { user: User }) {
   const images = usePoll<Image[]>('/api/images', 30_000);
   const disk = usePoll<Disk>('/api/disk', 30_000);
   const [pruning, setPruning] = useState(false);
-  const [reclaimed, setReclaimed] = useState<number>();
+  const [result, setResult] = useState<{ reclaimed: number; removed: number }>();
 
   if (images.error && !images.data) {
     return <ErrorState title="The console could not list images">{images.error.message}</ErrorState>;
@@ -19,15 +19,15 @@ export function Images({ user }: { user: User }) {
       <PageHeader
         actions={
           can(user, 'area:images') && (
-            <Button onClick={() => setPruning(true)}>Remove dangling images</Button>
+            <Button onClick={() => setPruning(true)}>Remove unused images</Button>
           )
         }
         title="Images"
       >
         {disk.data && (
           <span className="mono-sm text-muted">
             images {bytes(disk.data.images)} · containers {bytes(disk.data.containers)} · build cache {bytes(disk.data.buildCache)}
-            {reclaimed !== undefined && ` · reclaimed ${bytes(reclaimed)}`}
+            {result && ` · removed ${result.removed} images, reclaimed ${bytes(result.reclaimed)}`}
           </span>
         )}
       </PageHeader>
@@ -59,13 +59,12 @@ export function Images({ user }: { user: User }) {
           confirmLabel="Remove"
           onClose={() => setPruning(false)}
           onConfirm={async () => {
-            const result = await post<{ reclaimed: number }>('/api/prune');
-            setReclaimed(result.reclaimed);
-            await images.reload();
+            setResult(await post<{ reclaimed: number; removed: number }>('/api/prune'));
+            await Promise.all([images.reload(), disk.reload()]);
           }}
-          title="Remove dangling images?"
+          title={`Remove ${unused.length} unused images?`}
         >
-          Removes untagged image layers no container uses. Tagged images stay.
+          Removes every image no container uses, running or stopped, older builds included. A project pulls its image again when it next deploys.
         </ConfirmDialog>
       )}
     </div>