fix: harden the deploy agent and its trigger, back up volumes through container labels

.github/workflows/deploy.yml

@@ -7,7 +7,6 @@ on:
       - deploy/**
       - edge/**
       - host/**
-      - projects/**
       - shared/**
   workflow_dispatch:
     inputs:
@@ -41,6 +40,21 @@ jobs:
       - uses: actions/checkout@v7
         with:
           fetch-depth: 0
+      - name: Allow automated pushes to change only image tags
+        if: github.event_name == 'push' && endsWith(github.actor, '[bot]')
+        run: |
+          set -euo pipefail
+          if ! git cat-file -e "${BEFORE}^{commit}" 2>/dev/null; then
+            echo "::error::${GITHUB_ACTOR} pushed without a previous commit to compare against; deploy by hand after review."
+            exit 1
+          fi
+          files="$(git diff --name-only "$BEFORE" HEAD | grep -Ev '^deploy/[a-z0-9-]+/values$' || true)"
+          lines="$(git diff -U0 "$BEFORE" HEAD -- 'deploy/*/values' | grep -E '^[+-]' | grep -Ev '^(\+\+\+|---) ' | grep -Ev '^[+-]IMAGE_TAG=sha-[0-9a-f]{7,40}$' || true)"
+          if [ -n "$files" ] || [ -n "$lines" ]; then
+            echo "::error::${GITHUB_ACTOR} may change only IMAGE_TAG lines in deploy/*/values; refusing to deploy:"
+            printf '%s\n' "$files" "$lines"
+            exit 1
+          fi
       - uses: actions/setup-python@v7
         with:
           python-version: "3.14"

deploy/console/compose.yaml

@@ -12,15 +12,19 @@ services:
       ACCESS_AUD: ${ACCESS_AUD:?}
       ALERTS_TOKEN: ${ALERTS_TOKEN:-}
       SECRETS_KEY: ${SECRETS_KEY:-}
+      SMTP_URL: ${SMTP_URL:-}
+      ALERTS_EMAIL: ${ALERTS_EMAIL:-}
+      ALERTS_FROM: ${ALERTS_FROM:-}
     privileged: true
     pid: host
     volumes: [/var/run/docker.sock:/var/run/docker.sock, data:/data]
+    labels: {graphite.backup.volumes: data}
     healthcheck:
       test: [CMD, node, -e, "fetch('http://127.0.0.1:3000/healthz').then((r) => process.exit(r.ok ? 0 : 1), () => process.exit(1))"]
       start_period: 10s
 
 volumes:
-  data: {labels: {graphite.backup: "true"}}
+  data: {}
 
 networks:
   edge: {external: true}

deploy/strata/compose.yaml

@@ -23,6 +23,7 @@ services:
       VAPID_PRIVATE_KEY: ${VAPID_PRIVATE_KEY:-}
       VAPID_SUBJECT: ${VAPID_SUBJECT:-}
     volumes: [files-data:/data/files]
+    labels: {graphite.backup.volumes: files-data}
     healthcheck:
       test: [CMD-SHELL, "wget -q -O /dev/null http://127.0.0.1:3000/api/v1/health || curl -fsS -o /dev/null http://127.0.0.1:3000/api/v1/health"]
       start_period: 20s
@@ -37,7 +38,7 @@ services:
     extends: {file: ../../shared/redis-password.yaml, service: redis}
 
 volumes:
-  files-data: {labels: {graphite.backup: "true"}}
+  files-data: {}
   postgres-data: {}
   redis-data: {}
 

edge/agent.yml

@@ -1,7 +1,7 @@
 http:
   routers:
     agent:
-      rule: Host(`deploy.project-graphite.com`)
+      rule: Host(`deploy.project-graphite.com`) && (Path(`/deploy`) || Path(`/healthz`))
       entryPoints: [websecure]
       service: agent
       tls:

edge/backup.py

@@ -186,9 +186,12 @@ def targets():
     for container in docker("GET", filtered("/containers/json", label=["graphite.backup.postgres=true"])):
         project = container["Labels"].get("com.docker.compose.project", "other")
         found.setdefault(project, {"postgres": [], "volumes": []})["postgres"].append(container)
-    for volume in docker("GET", filtered("/volumes", label=["graphite.backup=true"]))["Volumes"] or []:
-        project = (volume.get("Labels") or {}).get("com.docker.compose.project", "other")
-        found.setdefault(project, {"postgres": [], "volumes": []})["volumes"].append(volume)
+    for container in docker("GET", filtered("/containers/json", label=["graphite.backup.volumes"])):
+        project = container["Labels"].get("com.docker.compose.project", "other")
+        volumes = found.setdefault(project, {"postgres": [], "volumes": []})["volumes"]
+        for name in filter(None, (part.strip() for part in container["Labels"]["graphite.backup.volumes"].split(","))):
+            if all(volume["Name"] != f"{project}_{name}" for volume in volumes):
+                volumes.append({"Name": f"{project}_{name}", "Labels": {"com.docker.compose.volume": name}})
     return found
 
 

host/agent.py

@@ -5,35 +5,61 @@
 import pathlib
 import re
 import subprocess
+import threading
 import time
-from http.server import BaseHTTPRequestHandler, HTTPServer
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 
 BASE = pathlib.Path(os.environ.get("GRAPHITE_HOME", "/srv/graphite"))
 TOKEN = os.environ["DEPLOY_TOKEN"].encode()
-CONSOLE_TOKEN = os.environ.get("CONSOLE_TOKEN", "").encode()
 ENV = pathlib.Path("/etc/graphite")
 NAME = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,38}[a-z0-9])?$")
 FILES = re.compile(r"^(compose\.yaml|traefik\.yml|values|files/[A-Za-z0-9._-]+)$")
+HOST_ACCESS = {"console", "edge", "backup"}
+HOST_SETTINGS = (
+    "privileged", "pid", "ipc", "uts", "userns_mode", "network_mode", "cgroup", "cgroup_parent",
+    "cap_add", "devices", "device_cgroup_rules", "volumes_from", "sysctls", "ports", "build", "env_file",
+)
+RULE = re.compile(r"^Host\(`[a-z0-9.-]+`\)(?: \|\| Host\(`[a-z0-9.-]+`\))*$")
+HOSTS = re.compile(r"Host\(`([^`]+)`\)")
 SKEW = 300
+MAX_BODY = 4 << 20
+CLEAN = {"PATH": os.environ.get("PATH", "/usr/sbin:/usr/bin:/sbin:/bin"), "HOME": os.environ.get("HOME", "/root")}
+LOCK = threading.Lock()
+SEEN_LOCK = threading.Lock()
+SEEN = {}
+
+
+class Refused(Exception):
+    pass
 
 
 def signed(stamp, signature, body):
     if not stamp.isdigit() or abs(time.time() - int(stamp)) > SKEW:
         return False
     expected = hmac.new(TOKEN, stamp.encode() + b"." + body, hashlib.sha256).hexdigest()
-    return hmac.compare_digest(expected, signature)
+    if not hmac.compare_digest(expected, signature):
+        return False
+    now = time.time()
+    with SEEN_LOCK:
+        for seen, until in list(SEEN.items()):
+            if until < now:
+                del SEEN[seen]
+        if signature in SEEN:
+            return False
+        SEEN[signature] = now + 2 * SKEW
+    return True
 
 
 def write(path, text):
     path.parent.mkdir(parents=True, exist_ok=True)
     temporary = path.with_name(path.name + ".new")
     temporary.write_text(text, encoding="utf-8")
-    temporary.chmod(0o600 if path.name == "secrets.env" else 0o644)
+    temporary.chmod(0o644)
     temporary.replace(path)
 
 
 def run(*command):
-    result = subprocess.run(command, capture_output=True, text=True, timeout=600)
+    result = subprocess.run(command, capture_output=True, text=True, timeout=600, env=CLEAN)
     output = (result.stdout + result.stderr)[-4000:]
     if result.returncode:
         raise RuntimeError(output.strip() or f"{command[0]} failed")
@@ -56,11 +82,70 @@ def compose_options(name):
     return [*(["--env-file", str(values)] if values.exists() else []), "--env-file", str(env_file(name))]
 
 
+def inside(path):
+    return isinstance(path, str) and path.startswith("./") and ".." not in pathlib.PurePosixPath(path).parts
+
+
+def compose_problems(name, text):
+    try:
+        compose = json.loads(text)
+    except ValueError:
+        return ["compose.yaml is not the JSON deploy.py renders"]
+    problems = []
+    for service_name, service in (compose.get("services") or {}).items():
+        where = f"services.{service_name}"
+        problems.extend(f"{where}.{key}" for key in HOST_SETTINGS if service.get(key))
+        if any(option != "no-new-privileges:true" for option in service.get("security_opt") or []):
+            problems.append(f"{where}.security_opt")
+        for volume in service.get("volumes") or []:
+            if volume.get("type") not in ("volume", "bind", "tmpfs") or (volume.get("type") == "bind" and not inside(volume.get("source"))):
+                problems.append(f"{where}.volumes {volume.get('source')}")
+    for key in ("configs", "secrets"):
+        for entry_name, entry in (compose.get(key) or {}).items():
+            if entry.get("external") or entry.get("environment") or ("file" in entry and not inside(entry["file"])):
+                problems.append(f"{key}.{entry_name}")
+    for volume_name, volume in (compose.get("volumes") or {}).items():
+        if volume.get("external") or volume.get("driver_opts") or volume.get("driver") not in (None, "local") or not str(volume.get("name", f"{name}_")).startswith(f"{name}_"):
+            problems.append(f"volumes.{volume_name}")
+    for network_name, network in (compose.get("networks") or {}).items():
+        if network.get("driver_opts") or (network.get("external") and network.get("name", network_name) != "edge"):
+            problems.append(f"networks.{network_name}")
+    return problems
+
+
+def route_problems(name, text):
+    try:
+        http = json.loads(text)["http"]
+    except (ValueError, KeyError, TypeError):
+        return ["the route is not the JSON deploy.py renders"]
+    problems = [f"route {key}" for key in http if key not in ("routers", "services")]
+    for router_name, router in http.get("routers", {}).items():
+        if set(router) - {"rule", "entryPoints", "service", "tls"} or not RULE.match(router.get("rule", "")):
+            problems.append(f"router {router_name}")
+    for service_name, service in http.get("services", {}).items():
+        for server in service.get("loadBalancer", {}).get("servers", []):
+            backend = re.match(r"^http://([a-z0-9-]+):[0-9]+$", server.get("url", ""))
+            if not backend or not (backend[1] == name or backend[1].startswith(f"{name}-")):
+                problems.append(f"service {service_name} sends traffic to {server.get('url')}")
+    claimed = set(HOSTS.findall(text))
+    for other in (BASE / "edge" / "routes").glob("*.yml"):
+        taken = claimed & set(HOSTS.findall(other.read_text(encoding="utf-8")))
+        if other.stem != name and taken:
+            problems.append(f"{', '.join(sorted(taken))} already belongs to {other.stem}")
+    return problems
+
+
 def up(name, request):
     folder = BASE / name
     unknown = sorted(file for file in request.get("files", {}) if not FILES.match(file))
     if unknown:
         raise ValueError(f"{', '.join(unknown)} is not a file the agent writes")
+    if name not in HOST_ACCESS:
+        problems = compose_problems(name, request.get("files", {}).get("compose.yaml") or "")
+        if request.get("route") is not None:
+            problems += route_problems(name, request["route"])
+        if problems:
+            raise Refused(f"{name} may not use: {'; '.join(problems)}")
     missing = sorted(set(request.get("require", [])) - keys(env_file(name)))
     if missing:
         raise LookupError(f"{name} is waiting on secrets: {', '.join(missing)}")
@@ -87,21 +172,9 @@ def down(name):
     return run("docker", "compose", "--project-directory", str(folder), "-p", name, *compose_options(name), "down", "--remove-orphans")
 
 
-def reapply(name):
-    if not (BASE / name / "compose.yaml").exists():
-        raise LookupError(f"{name} is not deployed")
-    return run(str(BASE / "bin" / "graphite-up"), name)
-
-
-def secrets(name, request):
-    folder = BASE / name
-    write(folder / "secrets.env", request["secrets"])
-    if not (folder / "compose.yaml").exists():
-        return f"Stored the secrets for {name}; it is not deployed yet."
-    return run(str(BASE / "bin" / "graphite-up"), name)
-
-
 class Agent(BaseHTTPRequestHandler):
+    timeout = 30
+
     def reply(self, status, body):
         data = json.dumps(body).encode()
         self.send_response(status)
@@ -114,33 +187,30 @@ def do_GET(self):
         self.reply(200 if self.path == "/healthz" else 404, {"ok": self.path == "/healthz"})
 
     def do_POST(self):
-        body = self.rfile.read(int(self.headers.get("Content-Length", 0)))
-        if self.path == "/reapply":
-            allowed = CONSOLE_TOKEN and hmac.compare_digest(self.headers.get("Authorization", "").encode(), b"Bearer " + CONSOLE_TOKEN)
-        else:
-            allowed = self.path == "/deploy" and signed(
-                self.headers.get("X-Graphite-Timestamp", ""), self.headers.get("X-Graphite-Signature", ""), body
-            )
-        if not allowed:
-            self.reply(401, {"error": "unsigned or expired request"})
+        length = self.headers.get("Content-Length", "")
+        if self.path != "/deploy" or not length.isdigit() or int(length) > MAX_BODY:
+            self.reply(404 if self.path != "/deploy" else 413, {"error": "not found" if self.path != "/deploy" else "the request is too large"})
+            return
+        body = self.rfile.read(int(length))
+        if not signed(self.headers.get("X-Graphite-Timestamp", ""), self.headers.get("X-Graphite-Signature", ""), body):
+            self.reply(401, {"error": "unsigned, expired or repeated request"})
             return
         request = json.loads(body)
-        name, action = request.get("name", ""), "reapply" if self.path == "/reapply" else request.get("action")
+        name, action = request.get("name", ""), request.get("action")
         if not NAME.match(name):
             self.reply(400, {"error": f"{name!r} is not a project name"})
             return
         try:
-            if action == "up":
-                output = up(name, request)
-            elif action == "down":
-                output = down(name)
-            elif action == "secrets":
-                output = secrets(name, request)
-            elif action == "reapply":
-                output = reapply(name)
-            else:
-                self.reply(400, {"error": f"unknown action {action!r}"})
-                return
+            with LOCK:
+                if action == "up":
+                    output = up(name, request)
+                elif action == "down":
+                    output = down(name)
+                else:
+                    self.reply(400, {"error": f"unknown action {action!r}"})
+                    return
+        except Refused as error:
+            self.reply(403, {"error": str(error)})
         except LookupError as error:
             self.reply(409, {"error": str(error)})
         except (RuntimeError, ValueError, subprocess.TimeoutExpired) as error:
@@ -150,4 +220,4 @@ def do_POST(self):
 
 
 if __name__ == "__main__":
-    HTTPServer((os.environ.get("BIND", "172.17.0.1"), 9000), Agent).serve_forever()
+    ThreadingHTTPServer((os.environ.get("BIND", "172.17.0.1"), 9000), Agent).serve_forever()

host/graphite-up

@@ -14,4 +14,5 @@ if [ -f values ]; then
   set -- --env-file values
 fi
 docker compose -p "$name" "$@" --env-file "$env" up -d --wait --wait-timeout 300 --remove-orphans
+touch .applied
 docker image prune -af --filter until=24h >/dev/null || echo "graphite-up: image prune failed" >&2

scripts/agent.py

@@ -33,10 +33,3 @@ def send(payload, attempts=6):
                 raise SystemExit(f"error: could not reach the deploy agent: {error}") from error
             print(f"The deploy agent is not reachable yet ({error}); retrying ({attempt}/{attempts}).", file=sys.stderr)
             time.sleep(10)
-
-
-if __name__ == "__main__":
-    if len(sys.argv) != 3 or sys.argv[1] != "secrets":
-        print("usage: agent.py secrets <name> < secrets.env", file=sys.stderr)
-        sys.exit(1)
-    print(send({"name": sys.argv[2], "action": "secrets", "secrets": sys.stdin.read()}))

scripts/check-values.py

@@ -6,7 +6,7 @@
 REFERENCE = re.compile(r"(?<![$])[$][{]([^}]*)[}]")
 SHAPE = re.compile(r"[A-Z_][A-Z0-9_]*(?::?[?].*|:-.*)")
 DEFAULT = re.compile(r"(?<![$])[$][{]([A-Z_][A-Z0-9_]*):-([^}]*)[}]")
-IMAGE = re.compile(r"^\s*image:\s*(ghcr\.io/\S+)\s*$", re.MULTILINE)
+IMAGE = re.compile(r'"image":\s*"(ghcr\.io/[^"]+)"')
 SUPPLIED = {"PROJECT", "HOST"}
 
 

scripts/deploy.py

@@ -73,7 +73,7 @@ def compose(slug, *args, env=()):
 
 
 def flatten(slug):
-    return compose(slug, "config", "--no-interpolate").replace(str(DEPLOY / slug) + os.sep, "./")
+    return compose(slug, "config", "--no-interpolate", "--format", "json").replace(str(DEPLOY / slug) + os.sep, "./")
 
 
 def required(text):
@@ -91,12 +91,12 @@ def route(slug, config):
     host = entry(slug)["deploy"]["host"]
     for service in config["services"].values():
         if slug in (((service.get("networks") or {}).get("edge") or {}).get("aliases") or []):
-            return yaml.safe_dump({
+            return json.dumps({
                 "http": {
                     "routers": {slug: {"rule": f"Host(`{host}`)", "entryPoints": ["websecure"], "service": slug, "tls": {"certResolver": "letsencrypt"}}},
                     "services": {slug: {"loadBalancer": {"servers": [{"url": f"http://{slug}:{service['expose'][0]}"}]}}},
                 }
-            }, sort_keys=False)
+            }, indent=2)
     raise ValueError(f"deploy/{slug}: no service joins the edge network as {slug}")