From c0183849f3a951e5d18830ff50912bbf07390794 Mon Sep 17 00:00:00 2001 From: Amr Mohammed El-Sheraey <141947355+AmrMsCLL@users.noreply.github.com> Date: Thu, 5 Nov 2026 04:47:56 +0200 Subject: [PATCH 1/3] fix: harden the deploy agent and its trigger, and back up volumes through container labels The agent caps the request body, times out stalled clients, serves health while a deploy runs, refuses a replayed signature, and drops /reapply, CONSOLE_TOKEN and the secrets action. graphite-up and compose run without the agent's environment. Every project but console, edge and backup gets a policy: no privileged or host settings, no host paths, ports, env_file or foreign volumes and networks, and a route only for its own hostnames and containers. deploy.py sends compose and the route as JSON for it. The public route to the agent passes only /deploy and /healthz. A bot push may change only IMAGE_TAG lines before Deploy runs. Existing volumes cannot take new labels, so the backup service reads the volumes to keep from graphite.backup.volumes on the container that mounts them. --- .github/workflows/deploy.yml | 16 +++- deploy/console/compose.yaml | 3 +- deploy/strata/compose.yaml | 3 +- edge/agent.yml | 2 +- edge/backup.py | 9 ++- host/agent.py | 152 +++++++++++++++++++++++++---------- scripts/agent.py | 7 -- scripts/check-values.py | 2 +- scripts/deploy.py | 6 +- 9 files changed, 141 insertions(+), 59 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 65a2bdc..7c7430d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -7,7 +7,6 @@ on: - deploy/** - edge/** - host/** - - projects/** - shared/** workflow_dispatch: inputs: @@ -41,6 +40,21 @@ jobs: - uses: actions/checkout@v7 with: fetch-depth: 0 + - name: Allow automated pushes to change only image tags + if: github.event_name == 'push' && endsWith(github.actor, '[bot]') + run: | + set -euo pipefail + if ! git cat-file -e "${BEFORE}^{commit}" 2>/dev/null; then + echo "::error::${GITHUB_ACTOR} pushed without a previous commit to compare against; deploy by hand after review." + exit 1 + fi + files="$(git diff --name-only "$BEFORE" HEAD | grep -Ev '^deploy/[a-z0-9-]+/values$' || true)" + lines="$(git diff -U0 "$BEFORE" HEAD -- 'deploy/*/values' | grep -E '^[+-]' | grep -Ev '^(\+\+\+|---) ' | grep -Ev '^[+-]IMAGE_TAG=sha-[0-9a-f]{7,40}$' || true)" + if [ -n "$files" ] || [ -n "$lines" ]; then + echo "::error::${GITHUB_ACTOR} may change only IMAGE_TAG lines in deploy/*/values; refusing to deploy:" + printf '%s\n' "$files" "$lines" + exit 1 + fi - uses: actions/setup-python@v7 with: python-version: "3.14" diff --git a/deploy/console/compose.yaml b/deploy/console/compose.yaml index 45a85a2..76c25ee 100644 --- a/deploy/console/compose.yaml +++ b/deploy/console/compose.yaml @@ -15,12 +15,13 @@ services: privileged: true pid: host volumes: [/var/run/docker.sock:/var/run/docker.sock, data:/data] + labels: {graphite.backup.volumes: data} healthcheck: test: [CMD, node, -e, "fetch('http://127.0.0.1:3000/healthz').then((r) => process.exit(r.ok ? 0 : 1), () => process.exit(1))"] start_period: 10s volumes: - data: {labels: {graphite.backup: "true"}} + data: {} networks: edge: {external: true} diff --git a/deploy/strata/compose.yaml b/deploy/strata/compose.yaml index d53bdd0..14ee823 100644 --- a/deploy/strata/compose.yaml +++ b/deploy/strata/compose.yaml @@ -23,6 +23,7 @@ services: VAPID_PRIVATE_KEY: ${VAPID_PRIVATE_KEY:-} VAPID_SUBJECT: ${VAPID_SUBJECT:-} volumes: [files-data:/data/files] + labels: {graphite.backup.volumes: files-data} healthcheck: test: [CMD-SHELL, "wget -q -O /dev/null http://127.0.0.1:3000/api/v1/health || curl -fsS -o /dev/null http://127.0.0.1:3000/api/v1/health"] start_period: 20s @@ -37,7 +38,7 @@ services: extends: {file: ../../shared/redis-password.yaml, service: redis} volumes: - files-data: {labels: {graphite.backup: "true"}} + files-data: {} postgres-data: {} redis-data: {} diff --git a/edge/agent.yml b/edge/agent.yml index aec9001..2844513 100644 --- a/edge/agent.yml +++ b/edge/agent.yml @@ -1,7 +1,7 @@ http: routers: agent: - rule: Host(`deploy.project-graphite.com`) + rule: Host(`deploy.project-graphite.com`) && (Path(`/deploy`) || Path(`/healthz`)) entryPoints: [websecure] service: agent tls: diff --git a/edge/backup.py b/edge/backup.py index e67126d..35d6535 100644 --- a/edge/backup.py +++ b/edge/backup.py @@ -186,9 +186,12 @@ def targets(): for container in docker("GET", filtered("/containers/json", label=["graphite.backup.postgres=true"])): project = container["Labels"].get("com.docker.compose.project", "other") found.setdefault(project, {"postgres": [], "volumes": []})["postgres"].append(container) - for volume in docker("GET", filtered("/volumes", label=["graphite.backup=true"]))["Volumes"] or []: - project = (volume.get("Labels") or {}).get("com.docker.compose.project", "other") - found.setdefault(project, {"postgres": [], "volumes": []})["volumes"].append(volume) + for container in docker("GET", filtered("/containers/json", label=["graphite.backup.volumes"])): + project = container["Labels"].get("com.docker.compose.project", "other") + volumes = found.setdefault(project, {"postgres": [], "volumes": []})["volumes"] + for name in filter(None, (part.strip() for part in container["Labels"]["graphite.backup.volumes"].split(","))): + if all(volume["Name"] != f"{project}_{name}" for volume in volumes): + volumes.append({"Name": f"{project}_{name}", "Labels": {"com.docker.compose.volume": name}}) return found diff --git a/host/agent.py b/host/agent.py index 7ba86b5..286a4c6 100644 --- a/host/agent.py +++ b/host/agent.py @@ -5,35 +5,61 @@ import pathlib import re import subprocess +import threading import time -from http.server import BaseHTTPRequestHandler, HTTPServer +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer BASE = pathlib.Path(os.environ.get("GRAPHITE_HOME", "/srv/graphite")) TOKEN = os.environ["DEPLOY_TOKEN"].encode() -CONSOLE_TOKEN = os.environ.get("CONSOLE_TOKEN", "").encode() ENV = pathlib.Path("/etc/graphite") NAME = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,38}[a-z0-9])?$") FILES = re.compile(r"^(compose\.yaml|traefik\.yml|values|files/[A-Za-z0-9._-]+)$") +HOST_ACCESS = {"console", "edge", "backup"} +HOST_SETTINGS = ( + "privileged", "pid", "ipc", "uts", "userns_mode", "network_mode", "cgroup", "cgroup_parent", + "cap_add", "devices", "device_cgroup_rules", "volumes_from", "sysctls", "ports", "build", "env_file", +) +RULE = re.compile(r"^Host\(`[a-z0-9.-]+`\)(?: \|\| Host\(`[a-z0-9.-]+`\))*$") +HOSTS = re.compile(r"Host\(`([^`]+)`\)") SKEW = 300 +MAX_BODY = 4 << 20 +CLEAN = {"PATH": os.environ.get("PATH", "/usr/sbin:/usr/bin:/sbin:/bin"), "HOME": os.environ.get("HOME", "/root")} +LOCK = threading.Lock() +SEEN_LOCK = threading.Lock() +SEEN = {} + + +class Refused(Exception): + pass def signed(stamp, signature, body): if not stamp.isdigit() or abs(time.time() - int(stamp)) > SKEW: return False expected = hmac.new(TOKEN, stamp.encode() + b"." + body, hashlib.sha256).hexdigest() - return hmac.compare_digest(expected, signature) + if not hmac.compare_digest(expected, signature): + return False + now = time.time() + with SEEN_LOCK: + for seen, until in list(SEEN.items()): + if until < now: + del SEEN[seen] + if signature in SEEN: + return False + SEEN[signature] = now + 2 * SKEW + return True def write(path, text): path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(path.name + ".new") temporary.write_text(text, encoding="utf-8") - temporary.chmod(0o600 if path.name == "secrets.env" else 0o644) + temporary.chmod(0o644) temporary.replace(path) def run(*command): - result = subprocess.run(command, capture_output=True, text=True, timeout=600) + result = subprocess.run(command, capture_output=True, text=True, timeout=600, env=CLEAN) output = (result.stdout + result.stderr)[-4000:] if result.returncode: raise RuntimeError(output.strip() or f"{command[0]} failed") @@ -56,11 +82,70 @@ def compose_options(name): return [*(["--env-file", str(values)] if values.exists() else []), "--env-file", str(env_file(name))] +def inside(path): + return isinstance(path, str) and path.startswith("./") and ".." not in pathlib.PurePosixPath(path).parts + + +def compose_problems(name, text): + try: + compose = json.loads(text) + except ValueError: + return ["compose.yaml is not the JSON deploy.py renders"] + problems = [] + for service_name, service in (compose.get("services") or {}).items(): + where = f"services.{service_name}" + problems.extend(f"{where}.{key}" for key in HOST_SETTINGS if service.get(key)) + if any(option != "no-new-privileges:true" for option in service.get("security_opt") or []): + problems.append(f"{where}.security_opt") + for volume in service.get("volumes") or []: + if volume.get("type") not in ("volume", "bind", "tmpfs") or (volume.get("type") == "bind" and not inside(volume.get("source"))): + problems.append(f"{where}.volumes {volume.get('source')}") + for key in ("configs", "secrets"): + for entry_name, entry in (compose.get(key) or {}).items(): + if entry.get("external") or entry.get("environment") or ("file" in entry and not inside(entry["file"])): + problems.append(f"{key}.{entry_name}") + for volume_name, volume in (compose.get("volumes") or {}).items(): + if volume.get("external") or volume.get("driver_opts") or volume.get("driver") not in (None, "local") or not str(volume.get("name", f"{name}_")).startswith(f"{name}_"): + problems.append(f"volumes.{volume_name}") + for network_name, network in (compose.get("networks") or {}).items(): + if network.get("driver_opts") or (network.get("external") and network.get("name", network_name) != "edge"): + problems.append(f"networks.{network_name}") + return problems + + +def route_problems(name, text): + try: + http = json.loads(text)["http"] + except (ValueError, KeyError, TypeError): + return ["the route is not the JSON deploy.py renders"] + problems = [f"route {key}" for key in http if key not in ("routers", "services")] + for router_name, router in http.get("routers", {}).items(): + if set(router) - {"rule", "entryPoints", "service", "tls"} or not RULE.match(router.get("rule", "")): + problems.append(f"router {router_name}") + for service_name, service in http.get("services", {}).items(): + for server in service.get("loadBalancer", {}).get("servers", []): + backend = re.match(r"^http://([a-z0-9-]+):[0-9]+$", server.get("url", "")) + if not backend or not (backend[1] == name or backend[1].startswith(f"{name}-")): + problems.append(f"service {service_name} sends traffic to {server.get('url')}") + claimed = set(HOSTS.findall(text)) + for other in (BASE / "edge" / "routes").glob("*.yml"): + taken = claimed & set(HOSTS.findall(other.read_text(encoding="utf-8"))) + if other.stem != name and taken: + problems.append(f"{', '.join(sorted(taken))} already belongs to {other.stem}") + return problems + + def up(name, request): folder = BASE / name unknown = sorted(file for file in request.get("files", {}) if not FILES.match(file)) if unknown: raise ValueError(f"{', '.join(unknown)} is not a file the agent writes") + if name not in HOST_ACCESS: + problems = compose_problems(name, request.get("files", {}).get("compose.yaml") or "") + if request.get("route") is not None: + problems += route_problems(name, request["route"]) + if problems: + raise Refused(f"{name} may not use: {'; '.join(problems)}") missing = sorted(set(request.get("require", [])) - keys(env_file(name))) if missing: raise LookupError(f"{name} is waiting on secrets: {', '.join(missing)}") @@ -87,21 +172,9 @@ def down(name): return run("docker", "compose", "--project-directory", str(folder), "-p", name, *compose_options(name), "down", "--remove-orphans") -def reapply(name): - if not (BASE / name / "compose.yaml").exists(): - raise LookupError(f"{name} is not deployed") - return run(str(BASE / "bin" / "graphite-up"), name) - - -def secrets(name, request): - folder = BASE / name - write(folder / "secrets.env", request["secrets"]) - if not (folder / "compose.yaml").exists(): - return f"Stored the secrets for {name}; it is not deployed yet." - return run(str(BASE / "bin" / "graphite-up"), name) - - class Agent(BaseHTTPRequestHandler): + timeout = 30 + def reply(self, status, body): data = json.dumps(body).encode() self.send_response(status) @@ -114,33 +187,30 @@ def do_GET(self): self.reply(200 if self.path == "/healthz" else 404, {"ok": self.path == "/healthz"}) def do_POST(self): - body = self.rfile.read(int(self.headers.get("Content-Length", 0))) - if self.path == "/reapply": - allowed = CONSOLE_TOKEN and hmac.compare_digest(self.headers.get("Authorization", "").encode(), b"Bearer " + CONSOLE_TOKEN) - else: - allowed = self.path == "/deploy" and signed( - self.headers.get("X-Graphite-Timestamp", ""), self.headers.get("X-Graphite-Signature", ""), body - ) - if not allowed: - self.reply(401, {"error": "unsigned or expired request"}) + length = self.headers.get("Content-Length", "") + if self.path != "/deploy" or not length.isdigit() or int(length) > MAX_BODY: + self.reply(404 if self.path != "/deploy" else 413, {"error": "not found" if self.path != "/deploy" else "the request is too large"}) + return + body = self.rfile.read(int(length)) + if not signed(self.headers.get("X-Graphite-Timestamp", ""), self.headers.get("X-Graphite-Signature", ""), body): + self.reply(401, {"error": "unsigned, expired or repeated request"}) return request = json.loads(body) - name, action = request.get("name", ""), "reapply" if self.path == "/reapply" else request.get("action") + name, action = request.get("name", ""), request.get("action") if not NAME.match(name): self.reply(400, {"error": f"{name!r} is not a project name"}) return try: - if action == "up": - output = up(name, request) - elif action == "down": - output = down(name) - elif action == "secrets": - output = secrets(name, request) - elif action == "reapply": - output = reapply(name) - else: - self.reply(400, {"error": f"unknown action {action!r}"}) - return + with LOCK: + if action == "up": + output = up(name, request) + elif action == "down": + output = down(name) + else: + self.reply(400, {"error": f"unknown action {action!r}"}) + return + except Refused as error: + self.reply(403, {"error": str(error)}) except LookupError as error: self.reply(409, {"error": str(error)}) except (RuntimeError, ValueError, subprocess.TimeoutExpired) as error: @@ -150,4 +220,4 @@ def do_POST(self): if __name__ == "__main__": - HTTPServer((os.environ.get("BIND", "172.17.0.1"), 9000), Agent).serve_forever() + ThreadingHTTPServer((os.environ.get("BIND", "172.17.0.1"), 9000), Agent).serve_forever() diff --git a/scripts/agent.py b/scripts/agent.py index 3d30dd7..dba4ebc 100644 --- a/scripts/agent.py +++ b/scripts/agent.py @@ -33,10 +33,3 @@ def send(payload, attempts=6): raise SystemExit(f"error: could not reach the deploy agent: {error}") from error print(f"The deploy agent is not reachable yet ({error}); retrying ({attempt}/{attempts}).", file=sys.stderr) time.sleep(10) - - -if __name__ == "__main__": - if len(sys.argv) != 3 or sys.argv[1] != "secrets": - print("usage: agent.py secrets < secrets.env", file=sys.stderr) - sys.exit(1) - print(send({"name": sys.argv[2], "action": "secrets", "secrets": sys.stdin.read()})) diff --git a/scripts/check-values.py b/scripts/check-values.py index 73dbd34..27b6619 100644 --- a/scripts/check-values.py +++ b/scripts/check-values.py @@ -6,7 +6,7 @@ REFERENCE = re.compile(r"(? Date: Thu, 5 Nov 2026 04:57:55 +0200 Subject: [PATCH 2/3] fix(host): graphite-up marks a successful rollout in .applied for the console's redeploy hint --- host/graphite-up | 1 + 1 file changed, 1 insertion(+) diff --git a/host/graphite-up b/host/graphite-up index d8a7311..74753af 100755 --- a/host/graphite-up +++ b/host/graphite-up @@ -14,4 +14,5 @@ if [ -f values ]; then set -- --env-file values fi docker compose -p "$name" "$@" --env-file "$env" up -d --wait --wait-timeout 300 --remove-orphans +touch .applied docker image prune -af --filter until=24h >/dev/null || echo "graphite-up: image prune failed" >&2 From 089352a2de02d9865e937bc4e03dfafc7eef1825 Mon Sep 17 00:00:00 2001 From: Amr Mohammed El-Sheraey <141947355+AmrMsCLL@users.noreply.github.com> Date: Thu, 5 Nov 2026 13:01:40 +0200 Subject: [PATCH 3/3] feat(console): pass SMTP_URL, ALERTS_EMAIL and ALERTS_FROM through for alert emails --- deploy/console/compose.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deploy/console/compose.yaml b/deploy/console/compose.yaml index 76c25ee..3bb9b23 100644 --- a/deploy/console/compose.yaml +++ b/deploy/console/compose.yaml @@ -12,6 +12,9 @@ services: ACCESS_AUD: ${ACCESS_AUD:?} ALERTS_TOKEN: ${ALERTS_TOKEN:-} SECRETS_KEY: ${SECRETS_KEY:-} + SMTP_URL: ${SMTP_URL:-} + ALERTS_EMAIL: ${ALERTS_EMAIL:-} + ALERTS_FROM: ${ALERTS_FROM:-} privileged: true pid: host volumes: [/var/run/docker.sock:/var/run/docker.sock, data:/data]