From 7c69c0c0f585c343c9ede3dfc928383c2696227b Mon Sep 17 00:00:00 2001 From: Amr Mohammed El-Sheraey <141947355+AmrMsCLL@users.noreply.github.com> Date: Thu, 5 Nov 2026 02:49:13 +0200 Subject: [PATCH 1/2] feat(host): projects read /etc/graphite/.env, the agent runs as root and re-applies for the console graphite-up and the agent use /etc/graphite/.env when it exists and secrets.env otherwise, so each project moves to the console's secret store on its next rollout after its secrets are imported. The agent runs as root, reads /etc/graphite/agent.env, and answers POST /reapply with a Bearer CONSOLE_TOKEN by re-running graphite-up on the project's last applied folder. --- host/agent.py | 31 +++++++++++++++++++++++++------ host/bootstrap.sh | 5 +++-- host/graphite-agent.service | 3 +-- host/graphite-up | 10 +++++++--- 4 files changed, 36 insertions(+), 13 deletions(-) diff --git a/host/agent.py b/host/agent.py index baeaa54..6d6c742 100644 --- a/host/agent.py +++ b/host/agent.py @@ -10,6 +10,8 @@ BASE = pathlib.Path(os.environ.get("GRAPHITE_HOME", "/srv/graphite")) TOKEN = os.environ["DEPLOY_TOKEN"].encode() +CONSOLE_TOKEN = os.environ.get("CONSOLE_TOKEN", "").encode() +ENV = pathlib.Path("/etc/graphite") NAME = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,38}[a-z0-9])?$") FILES = {"compose.yaml", "traefik.yml", "ofelia.ini", "sleep"} SKEW = 300 @@ -38,6 +40,11 @@ def run(*command): return output +def env_file(name): + path = ENV / f"{name}.env" + return path if path.exists() else BASE / name / "secrets.env" + + def keys(path): if not path.exists(): return set() @@ -54,7 +61,7 @@ def up(name, request): (folder / file).unlink(missing_ok=True) else: write(folder / file, content) - missing = sorted(set(request.get("require", [])) - keys(folder / "secrets.env")) + missing = sorted(set(request.get("require", [])) - keys(env_file(name))) if missing: raise LookupError(f"{name} is waiting on secrets: {', '.join(missing)}") output = run(str(BASE / "bin" / "graphite-up"), name) @@ -77,10 +84,16 @@ def down(name): return f"{name} is not deployed." return run( "docker", "compose", "--project-directory", str(folder), "-p", name, - "--env-file", str(folder / "secrets.env"), "down", "--remove-orphans", + "--env-file", str(env_file(name)), "down", "--remove-orphans", ) +def reapply(name): + if not (BASE / name / "compose.yaml").exists(): + raise LookupError(f"{name} is not deployed") + return run(str(BASE / "bin" / "graphite-up"), name) + + def secrets(name, request): folder = BASE / name write(folder / "secrets.env", request["secrets"]) @@ -103,13 +116,17 @@ def do_GET(self): def do_POST(self): body = self.rfile.read(int(self.headers.get("Content-Length", 0))) - if self.path != "/deploy" or not signed( - self.headers.get("X-Graphite-Timestamp", ""), self.headers.get("X-Graphite-Signature", ""), body - ): + if self.path == "/reapply": + allowed = CONSOLE_TOKEN and hmac.compare_digest(self.headers.get("Authorization", "").encode(), b"Bearer " + CONSOLE_TOKEN) + else: + allowed = self.path == "/deploy" and signed( + self.headers.get("X-Graphite-Timestamp", ""), self.headers.get("X-Graphite-Signature", ""), body + ) + if not allowed: self.reply(401, {"error": "unsigned or expired request"}) return request = json.loads(body) - name, action = request.get("name", ""), request.get("action") + name, action = request.get("name", ""), "reapply" if self.path == "/reapply" else request.get("action") if not NAME.match(name): self.reply(400, {"error": f"{name!r} is not a project name"}) return @@ -120,6 +137,8 @@ def do_POST(self): output = down(name) elif action == "secrets": output = secrets(name, request) + elif action == "reapply": + output = reapply(name) else: self.reply(400, {"error": f"unknown action {action!r}"}) return diff --git a/host/bootstrap.sh b/host/bootstrap.sh index d389129..b38d302 100755 --- a/host/bootstrap.sh +++ b/host/bootstrap.sh @@ -26,8 +26,9 @@ chmod 600 /srv/graphite/edge/secrets.env docker network inspect edge >/dev/null 2>&1 || docker network create edge bind="$(docker network inspect bridge -f '{{(index .IPAM.Config 0).Gateway}}')" -printf 'DEPLOY_TOKEN=%s\nBIND=%s\n' "$DEPLOY_TOKEN" "$bind" > /etc/graphite-agent.env -chmod 600 /etc/graphite-agent.env +install -d -m 700 /etc/graphite +printf 'DEPLOY_TOKEN=%s\nBIND=%s\n' "$DEPLOY_TOKEN" "$bind" > /etc/graphite/agent.env +chmod 600 /etc/graphite/agent.env systemctl daemon-reload systemctl enable --now graphite-agent diff --git a/host/graphite-agent.service b/host/graphite-agent.service index 77e0156..93f53a6 100644 --- a/host/graphite-agent.service +++ b/host/graphite-agent.service @@ -4,8 +4,7 @@ After=docker.service Requires=docker.service [Service] -User=deploy -EnvironmentFile=/etc/graphite-agent.env +EnvironmentFile=/etc/graphite/agent.env ExecStart=/usr/bin/python3 /srv/graphite/bin/agent.py Restart=always RestartSec=5 diff --git a/host/graphite-up b/host/graphite-up index 7fdec28..7200016 100755 --- a/host/graphite-up +++ b/host/graphite-up @@ -3,9 +3,13 @@ set -eu name=$1 cd "/srv/graphite/$name" -touch secrets.env -chmod 600 secrets.env -docker compose -p "$name" --env-file secrets.env up -d --wait --wait-timeout 300 --remove-orphans +env=/etc/graphite/$name.env +if [ ! -f "$env" ]; then + env=secrets.env + touch secrets.env + chmod 600 secrets.env +fi +docker compose -p "$name" --env-file "$env" up -d --wait --wait-timeout 300 --remove-orphans docker image prune -af --filter until=24h >/dev/null || echo "graphite-up: image prune failed" >&2 if [ -f sleep ]; then From 17321aa9acc64b70dfaccc954d555d6c34f238b5 Mon Sep 17 00:00:00 2001 From: Amr Mohammed El-Sheraey <141947355+AmrMsCLL@users.noreply.github.com> Date: Thu, 5 Nov 2026 03:28:40 +0200 Subject: [PATCH 2/2] feat: plain compose per project with shared fragments, a shared backup service, and the chart, tags and people retired Every compose project is deploy//compose.yaml extending shared/app.yaml, postgres.yaml, redis.yaml or redis-password.yaml, plus deploy//values with IMAGE_TAG and only the values that differ from a default. deploy.py flattens the compose, renders it with the secret names stubbed, and sends compose, values and files to the agent. check-memory.py and check-values.py run on every PR. edge/backup.yaml runs a nightly backup at 03:00 UTC of every postgres labelled graphite.backup.postgres and every volume labelled graphite.backup, with consistent copies of SQLite databases, verifies each archive, keeps 14 days, and runs, verifies and restores on request from the console. The core chart, deploy/chart.yaml, tags/, people/, compile.py, people.py, the person schema and the Main workflow are removed. validate.py checks owners against organisation membership. Message Lab and the disabled Traceboard lose their deploy folders, which takes Message Lab down. --- .github/workflows/deploy.yml | 9 +- .github/workflows/main.yml | 65 ---- .github/workflows/pr.yml | 17 +- .github/workflows/sync-registry.yml | 19 +- .gitignore | 1 - charts/core/Chart.yaml | 5 - charts/core/files/backup.sh | 16 - charts/core/templates/_helpers.tpl | 153 --------- charts/core/templates/compose.yaml | 134 -------- charts/core/templates/route.yaml | 59 ---- charts/core/values.schema.json | 152 --------- charts/core/values.yaml | 72 ----- deploy/chart.yaml | 1 - deploy/console/compose.yaml | 26 ++ deploy/console/core.yaml | 12 - deploy/console/values | 5 + deploy/console/values.yaml | 9 - deploy/digital-to-physical/values | 1 + deploy/digital-to-physical/values.yaml | 1 - deploy/graphite-hub/values | 1 + deploy/graphite-hub/values.yaml | 1 - deploy/graphite-message-lab/core.yaml | 26 -- deploy/graphite-message-lab/values.yaml | 18 -- deploy/graphite-tracker/compose.yaml | 47 +++ deploy/graphite-tracker/core.yaml | 10 - deploy/graphite-tracker/values | 10 + deploy/graphite-tracker/values.yaml | 18 -- deploy/pixelpocket/compose.yaml | 100 ++++++ deploy/pixelpocket/core.yaml | 35 -- deploy/pixelpocket/values | 5 + deploy/pixelpocket/values.yaml | 41 --- deploy/strata/compose.yaml | 45 +++ deploy/strata/core.yaml | 13 - deploy/strata/values | 8 + deploy/strata/values.yaml | 13 - deploy/traceboard/values.yaml | 1 - edge/backup.py | 403 ++++++++++++++++++++++++ edge/backup.yaml | 23 ++ edge/wake.yaml | 18 -- host/agent.py | 23 +- host/graphite-up | 11 +- people/amrmscll.yml | 32 -- people/bluocaroot.yml | 30 -- people/mahmoudhanyfathalla.yml | 7 - people/nier0o.yml | 7 - people/omar-mohamed24.yml | 34 -- schema/person.schema.json | 56 ---- scripts/check-memory.py | 48 +++ scripts/check-values.py | 58 ++++ scripts/compile.py | 105 ------ scripts/deploy.py | 355 ++++++--------------- scripts/people.py | 91 ------ scripts/sync.py | 68 +--- scripts/validate.py | 68 ++-- shared/app.yaml | 32 ++ shared/postgres.yaml | 20 ++ shared/redis-password.yaml | 8 + shared/redis.yaml | 14 + tags/console.json | 3 - tags/digital-to-physical.json | 3 - tags/graphite-hub.json | 3 - tags/graphite-message-lab.json | 4 - tags/graphite-tracker.json | 3 - tags/pixelpocket.json | 4 - tags/strata.json | 3 - 65 files changed, 1030 insertions(+), 1653 deletions(-) delete mode 100644 .github/workflows/main.yml delete mode 100644 charts/core/Chart.yaml delete mode 100644 charts/core/files/backup.sh delete mode 100644 charts/core/templates/_helpers.tpl delete mode 100644 charts/core/templates/compose.yaml delete mode 100644 charts/core/templates/route.yaml delete mode 100644 charts/core/values.schema.json delete mode 100644 charts/core/values.yaml delete mode 100644 deploy/chart.yaml create mode 100644 deploy/console/compose.yaml delete mode 100644 deploy/console/core.yaml create mode 100644 deploy/console/values delete mode 100644 deploy/console/values.yaml create mode 100644 deploy/digital-to-physical/values delete mode 100644 deploy/digital-to-physical/values.yaml create mode 100644 deploy/graphite-hub/values delete mode 100644 deploy/graphite-hub/values.yaml delete mode 100644 deploy/graphite-message-lab/core.yaml delete mode 100644 deploy/graphite-message-lab/values.yaml create mode 100644 deploy/graphite-tracker/compose.yaml delete mode 100644 deploy/graphite-tracker/core.yaml create mode 100644 deploy/graphite-tracker/values delete mode 100644 deploy/graphite-tracker/values.yaml create mode 100644 deploy/pixelpocket/compose.yaml delete mode 100644 deploy/pixelpocket/core.yaml create mode 100644 deploy/pixelpocket/values delete mode 100644 deploy/pixelpocket/values.yaml create mode 100644 deploy/strata/compose.yaml delete mode 100644 deploy/strata/core.yaml create mode 100644 deploy/strata/values delete mode 100644 deploy/strata/values.yaml delete mode 100644 deploy/traceboard/values.yaml create mode 100644 edge/backup.py create mode 100644 edge/backup.yaml delete mode 100644 edge/wake.yaml delete mode 100644 people/amrmscll.yml delete mode 100644 people/bluocaroot.yml delete mode 100644 people/mahmoudhanyfathalla.yml delete mode 100644 people/nier0o.yml delete mode 100644 people/omar-mohamed24.yml delete mode 100644 schema/person.schema.json create mode 100644 scripts/check-memory.py create mode 100644 scripts/check-values.py delete mode 100644 scripts/compile.py delete mode 100644 scripts/people.py create mode 100644 shared/app.yaml create mode 100644 shared/postgres.yaml create mode 100644 shared/redis-password.yaml create mode 100644 shared/redis.yaml delete mode 100644 tags/console.json delete mode 100644 tags/digital-to-physical.json delete mode 100644 tags/graphite-hub.json delete mode 100644 tags/graphite-message-lab.json delete mode 100644 tags/graphite-tracker.json delete mode 100644 tags/pixelpocket.json delete mode 100644 tags/strata.json diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index b7d400a..65a2bdc 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -4,12 +4,11 @@ on: push: branches: [main] paths: - - charts/core/** - deploy/** - edge/** - host/** - projects/** - - tags/** + - shared/** workflow_dispatch: inputs: project: @@ -36,8 +35,6 @@ jobs: plan: name: Plan runs-on: ubuntu-latest - permissions: - contents: write outputs: pages: ${{ steps.plan.outputs.pages }} steps: @@ -48,7 +45,6 @@ jobs: with: python-version: "3.14" - run: pip install pyyaml - - run: python scripts/deploy.py release-core - id: plan run: python scripts/deploy.py plan | tee -a "$GITHUB_OUTPUT" @@ -66,9 +62,6 @@ jobs: with: python-version: "3.14" - run: pip install pyyaml - - uses: azure/setup-helm@v5 - with: - version: v3.22.0 - run: python scripts/deploy.py apply env: DEPLOY_TOKEN: ${{ secrets.DEPLOY_TOKEN }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index afe1ce4..0000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Main - -on: - push: - branches: [main] - paths: - - projects/** - - people/** - - schema/** - - scripts/** - -concurrency: - group: ${{ github.workflow }} - cancel-in-progress: false - -jobs: - publish: - name: Publish registry - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - - uses: actions/setup-python@v7 - with: - python-version: "3.14" - - - run: pip install pyyaml jsonschema - - - run: python scripts/validate.py - - - run: python scripts/compile.py registry.json - - - uses: actions/create-github-app-token@v3 - id: app-token - with: - client-id: ${{ secrets.PLATFORM_APP_CLIENT_ID }} - private-key: ${{ secrets.PLATFORM_APP_PRIVATE_KEY }} - owner: ${{ github.repository_owner }} - repositories: graphite-hub - - - name: Sync to Graphite Hub - env: - TOKEN: ${{ steps.app-token.outputs.token }} - OWNER: ${{ github.repository_owner }} - run: | - set -euo pipefail - - git clone --quiet --depth 1 "https://x-access-token:${TOKEN}@github.com/${OWNER}/graphite-hub.git" /tmp/graphite-hub - install -D registry.json /tmp/graphite-hub/site/src/data/registry.json - - cd /tmp/graphite-hub - git config user.name "graphite-registry" - git config user.email "registry@project-graphite.com" - git add site/src/data/registry.json - - if git diff --cached --quiet; then - echo "Registry unchanged; Graphite Hub already current." - exit 0 - fi - - git commit --message "chore: sync registry from platform@${GITHUB_SHA::7}" - git push - echo "Graphite Hub updated." diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index d0687fd..cf985a4 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -16,25 +16,28 @@ jobs: with: python-version: "3.14" - run: pip install pyyaml jsonschema + - uses: actions/create-github-app-token@v3 + id: app-token + with: + client-id: ${{ secrets.PLATFORM_APP_CLIENT_ID }} + private-key: ${{ secrets.PLATFORM_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} - run: python scripts/validate.py + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} deploy: name: Deploy values runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - with: - fetch-depth: 0 - uses: actions/setup-python@v7 with: python-version: "3.14" - run: pip install pyyaml - - uses: azure/setup-helm@v5 - with: - version: v3.22.0 - run: python scripts/deploy.py check - env: - BASE_REF: ${{ github.base_ref }} + - run: python scripts/check-memory.py + - run: python scripts/check-values.py secrets: name: Secrets diff --git a/.github/workflows/sync-registry.yml b/.github/workflows/sync-registry.yml index e6bdb98..06afbee 100644 --- a/.github/workflows/sync-registry.yml +++ b/.github/workflows/sync-registry.yml @@ -4,6 +4,8 @@ on: workflow_dispatch: repository_dispatch: types: [sync-registry] + schedule: + - cron: "17 * * * *" concurrency: group: sync-registry @@ -44,14 +46,9 @@ jobs: GH_TOKEN: ${{ steps.app-token.outputs.token }} run: python scripts/sync.py - - name: Sync people from GitHub profiles - id: people - continue-on-error: true + - run: python scripts/validate.py env: GH_TOKEN: ${{ steps.app-token.outputs.token }} - run: python scripts/people.py - - - run: python scripts/validate.py - name: Open registry update env: @@ -60,7 +57,7 @@ jobs: WORKFLOW_TOKEN: ${{ github.token }} run: | set -euo pipefail - git add projects people + git add projects if git diff --cached --quiet; then echo "Registry already current." exit 0 @@ -99,9 +96,9 @@ jobs: fi file_count="$(gh pr view "$pr" --json files --jq '.files | length')" - invalid_paths="$(gh pr view "$pr" --json files --jq '[.files[].path | select(test("^(projects|people)/[^/]+\\.yml$") | not)] | join(", ")')" + invalid_paths="$(gh pr view "$pr" --json files --jq '[.files[].path | select(test("^projects/[^/]+\\.yml$") | not)] | join(", ")')" if [ "$file_count" -eq 0 ] || [ -n "$invalid_paths" ]; then - echo "::error::Registry PR #${pr} changes files outside projects/*.yml and people/*.yml: ${invalid_paths}" + echo "::error::Registry PR #${pr} changes files outside projects/*.yml: ${invalid_paths}" exit 1 fi @@ -129,7 +126,7 @@ jobs: gh pr merge "$pr" --squash --delete-branch --match-head-commit "$head_sha" - name: Report skipped manifests - if: steps.sync.outcome == 'failure' || steps.people.outcome == 'failure' + if: steps.sync.outcome == 'failure' run: | - echo "::error::The sync skipped invalid repository manifests or could not read GitHub profiles; see the Sync repository manifests and Sync people steps." + echo "::error::The sync skipped invalid repository manifests; see the Sync repository manifests step." exit 1 diff --git a/.gitignore b/.gitignore index 157ae76..c18dd8d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1 @@ __pycache__/ -registry.json diff --git a/charts/core/Chart.yaml b/charts/core/Chart.yaml deleted file mode 100644 index d0260a7..0000000 --- a/charts/core/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v2 -name: core -description: Renders a Project Graphite project (deploy//core.yaml and values.yaml) into a Docker Compose project and its route. -type: application -version: 2.0.0 diff --git a/charts/core/files/backup.sh b/charts/core/files/backup.sh deleted file mode 100644 index 5beba23..0000000 --- a/charts/core/files/backup.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -set -eu -while true; do - stamp=$(date -u +%Y%m%dT%H%M%SZ) - dump="/backups/$PGDATABASE-$stamp.dump" - pg_dump -Fc -f "$dump.partial" - pg_restore --list "$dump.partial" > /dev/null - mv "$dump.partial" "$dump" - for volume in /data/*; do - if [ -d "$volume" ]; then - tar -czf "/backups/$(basename "$volume")-$stamp.tar.gz" -C /data "$(basename "$volume")" - fi - done - find /backups -type f -mtime "+$BACKUP_KEEP_DAYS" -delete - sleep 86400 -done diff --git a/charts/core/templates/_helpers.tpl b/charts/core/templates/_helpers.tpl deleted file mode 100644 index 96e0262..0000000 --- a/charts/core/templates/_helpers.tpl +++ /dev/null @@ -1,153 +0,0 @@ -{{- define "core.image" -}} -{{- if or (contains ":" .service.image) (contains "@" .service.image) -}} -{{ .service.image }} -{{- else -}} -{{ .service.image }}:{{ .service.tag | default (index .tags .name) | default "latest" }} -{{- end -}} -{{- end -}} - -{{- define "core.megabytes" -}} -{{- $amount := regexFind "^[0-9]+" . | atoi -}} -{{- if hasSuffix "G" . }}{{ mul $amount 1024 }}{{ else }}{{ $amount }}{{ end -}} -{{- end -}} - -{{- define "core.database" -}} -{{- .Values.postgres.database | default (replace "-" "_" .Values.slug) -}} -{{- end -}} - -{{- define "core.postgres" -}} -{{- $postgres := .Values.postgres -}} -{{- $database := include "core.database" . -}} -{{- $user := $postgres.user | default $database -}} -image: {{ $postgres.image }} -{{- with $postgres.settings }} -command: - - postgres - {{- range $key, $value := . }} - - -c - - {{ printf "%s=%v" $key $value | toRawJson }} - {{- end }} -{{- end }} -env: - {{- toYaml (merge (dict "POSTGRES_DB" $database "POSTGRES_USER" $user "POSTGRES_PASSWORD" "${POSTGRES_PASSWORD}") $postgres.env) | nindent 2 }} -volumes: [postgres-data:/var/lib/postgresql] -health: - test: [CMD-SHELL, {{ printf "pg_isready -U %s -d %s" $user $database | toRawJson }}] - interval: 10s - retries: 10 -stopGracePeriod: 30s -memory: {{ $postgres.memory }} -cpus: {{ $postgres.cpus | quote }} -{{- with $postgres.reserve }} -reserve: {{ . }} -{{- end }} -{{- end -}} - -{{- define "core.redis" -}} -{{- $redis := .Values.redis -}} -{{- $maxmemory := $redis.maxmemory | default (printf "%dmb" (div (mul (include "core.megabytes" $redis.memory | atoi) 3) 4)) -}} -{{- $flags := list "--appendonly" "yes" "--maxmemory" $maxmemory "--maxmemory-policy" (ternary "allkeys-lru" "noeviction" (ne (toString $redis.evict) "false")) -}} -image: {{ $redis.image }} -{{- if $redis.password }} -command: [sh, -c, {{ printf "exec redis-server %s --requirepass \"$$REDIS_PASSWORD\"" (join " " $flags) | toRawJson }}] -env: - REDIS_PASSWORD: ${REDIS_PASSWORD} -health: - test: [CMD-SHELL, "REDISCLI_AUTH=\"$$REDIS_PASSWORD\" redis-cli ping | grep -q PONG"] - interval: 10s - retries: 10 -{{- else }} -command: {{ toRawJson (prepend $flags "redis-server") }} -health: - test: [CMD, redis-cli, ping] - interval: 10s - retries: 10 -{{- end }} -volumes: [redis-data:/data] -memory: {{ $redis.memory }} -cpus: {{ $redis.cpus | quote }} -{{- with $redis.reserve }} -reserve: {{ . }} -{{- end }} -{{- end -}} - -{{- define "core.backup" -}} -{{- $backup := .Values.backup -}} -{{- $database := include "core.database" . -}} -image: {{ .Values.postgres.image }} -command: [sh, /scripts/backup.sh] -env: - PGHOST: postgres - PGUSER: {{ .Values.postgres.user | default $database }} - PGDATABASE: {{ $database }} - PGPASSWORD: ${POSTGRES_PASSWORD} - BACKUP_KEEP_DAYS: {{ $backup.keepDays | quote }} -dependsOn: {postgres: healthy} -volumes: - - backups:/backups - {{- range $backup.include }} - - {{ printf "%s:/data/%s:ro" . . }} - {{- end }} -files: - /scripts/backup.sh: {{ $.Files.Get "files/backup.sh" | toRawJson }} -memory: {{ $backup.memory }} -cpus: {{ $backup.cpus | quote }} -{{- with $backup.reserve }} -reserve: {{ . }} -{{- end }} -{{- end -}} - -{{- define "core.services" -}} -{{- $values := .Values -}} -{{- $components := dict -}} -{{- if $values.postgres.enabled }}{{ $_ := set $components "postgres" (include "core.postgres" . | fromYaml) }}{{ end -}} -{{- if $values.redis.enabled }}{{ $_ := set $components "redis" (include "core.redis" . | fromYaml) }}{{ end -}} -{{- if $values.backup.enabled }}{{ $_ := set $components "backup" (include "core.backup" . | fromYaml) }}{{ end -}} -{{- $shared := dict -}} -{{- $waits := dict -}} -{{- if $values.postgres.enabled -}} -{{- $database := include "core.database" . -}} -{{- $user := $values.postgres.user | default $database -}} -{{- $_ := set $shared "DATABASE_URL" (printf "postgresql://%s:${POSTGRES_PASSWORD}@postgres:5432/%s" $user $database) -}} -{{- $_ := set $shared "POSTGRES_HOST" "postgres" -}} -{{- $_ := set $shared "POSTGRES_PORT" "5432" -}} -{{- $_ := set $shared "POSTGRES_DB" $database -}} -{{- $_ := set $shared "POSTGRES_USER" $user -}} -{{- $_ := set $shared "POSTGRES_PASSWORD" "${POSTGRES_PASSWORD}" -}} -{{- $_ := set $waits "postgres" "healthy" -}} -{{- end -}} -{{- if $values.redis.enabled -}} -{{- $_ := set $shared "REDIS_URL" (ternary "redis://:${REDIS_PASSWORD}@redis:6379" "redis://redis:6379" (eq (toString $values.redis.password) "true")) -}} -{{- $_ := set $waits "redis" "healthy" -}} -{{- end -}} -{{- $all := dict -}} -{{- range $name, $service := $values.services -}} -{{- $env := deepCopy $shared -}} -{{- range $service.optional }}{{ $_ := set $env . (printf "${%s:-}" .) }}{{ end -}} -{{- range $service.secrets }}{{ $_ := set $env . (printf "${%s}" .) }}{{ end -}} -{{- range $key, $value := $service.env }}{{ $_ := set $env $key (toString $value) }}{{ end -}} -{{- $_ := set $all $name (merge (dict "env" $env "dependsOn" (merge (deepCopy ($service.dependsOn | default dict)) $waits)) (omit $service "env" "dependsOn" "secrets" "optional")) -}} -{{- end -}} -{{- range $name, $component := $components }}{{ $_ := set $all $name $component }}{{ end -}} -{{- toJson $all -}} -{{- end -}} - -{{- define "core.hosted" -}} -{{- $names := list -}} -{{- range $name, $service := (include "core.services" . | fromJson) -}} -{{- if $service.hosts }}{{ if not $service.port }}{{ fail (printf "%s sets hosts, so it must set port" $name) }}{{ end }}{{ $names = append $names $name }}{{ end -}} -{{- end -}} -{{- toJson (dict "names" $names) -}} -{{- end -}} - -{{- define "core.public" -}} -{{- $services := include "core.services" . | fromJson -}} -{{- $marked := list -}} -{{- $listening := list -}} -{{- range $name, $service := $services -}} -{{- if $service.public }}{{ $marked = append $marked $name }}{{ end -}} -{{- if $service.port }}{{ $listening = append $listening $name }}{{ end -}} -{{- end -}} -{{- if gt (len $marked) 1 }}{{ fail (printf "only one service can set public: true, not %s" (join ", " $marked)) }}{{ end -}} -{{- if $marked }}{{ first $marked }}{{ else if eq (len $listening) 1 }}{{ first $listening }}{{ end -}} -{{- end -}} diff --git a/charts/core/templates/compose.yaml b/charts/core/templates/compose.yaml deleted file mode 100644 index 555b960..0000000 --- a/charts/core/templates/compose.yaml +++ /dev/null @@ -1,134 +0,0 @@ -{{- $values := .Values -}} -{{- $conditions := dict "healthy" "service_healthy" "started" "service_started" "completed" "service_completed_successfully" -}} -{{- $volumes := dict -}} -{{- $configs := list -}} -{{- $services := include "core.services" . | fromJson -}} -{{- $public := include "core.public" . -}} -name: {{ $values.slug }} - -services: -{{- range $name, $service := $services }} - {{- $health := $service.health }} - {{- if kindIs "string" $health }}{{ $health = dict "path" $health }}{{ end }} - {{- $memory := $service.memory | default $values.service.memory }} - {{ $name }}: - image: {{ include "core.image" (dict "name" $name "service" $service "tags" $values.tags) }} - {{- with $service.entrypoint }} - entrypoint: {{ toRawJson . }} - {{- end }} - {{- with $service.command }} - command: {{ toRawJson . }} - {{- end }} - {{- with $service.env }} - environment: - {{- range $key, $value := . }} - {{ $key }}: {{ toString $value | toRawJson }} - {{- end }} - {{- end }} - {{- with $service.dependsOn }} - depends_on: - {{- range $dependency, $state := . }} - {{ $dependency }}: - condition: {{ get $conditions $state }} - {{- end }} - {{- end }} - {{- with $service.port }} - expose: - - {{ toString . | toRawJson }} - {{- end }} - {{- if or (eq $name $public) $service.hosts }} - networks: - default: {} - edge: - aliases: - {{- if eq $name $public }} - - {{ $values.slug }} - {{- end }} - {{- if $service.hosts }} - - {{ printf "%s-%s" $values.slug $name }} - {{- end }} - {{- end }} - {{- with $service.volumes }} - volumes: - {{- range . }} - - {{ toRawJson . }} - {{- $volume := first (splitList ":" .) }} - {{- if regexMatch "^[A-Za-z0-9][A-Za-z0-9_.-]*$" $volume }}{{ $_ := set $volumes $volume true }}{{ end }} - {{- end }} - {{- end }} - {{- with $service.files }} - configs: - {{- range $target, $content := . }} - {{- $config := printf "%s-%s" $name (base $target | replace "." "-") }} - {{- $configs = append $configs (dict "name" $config "content" $content) }} - - source: {{ $config }} - target: {{ $target }} - {{- end }} - {{- end }} - {{- with $health }} - healthcheck: - {{- if .path }} - {{- $url := printf "http://127.0.0.1:%v%s" $service.port .path }} - test: [CMD-SHELL, {{ printf "wget -q -O /dev/null %s || curl -fsS -o /dev/null %s" $url $url | toRawJson }}] - {{- else }} - test: {{ toRawJson .test }} - {{- end }} - interval: {{ .interval | default "30s" }} - timeout: {{ .timeout | default "5s" }} - retries: {{ .retries | default 3 }} - start_period: {{ .startPeriod | default "0s" }} - {{- end }} - init: {{ ne (toString $service.init) "false" }} - restart: {{ ternary "no" "unless-stopped" (default false $service.once) | quote }} - logging: - driver: json-file - options: {max-size: 10m, max-file: "5"} - {{- with $service.stopGracePeriod }} - stop_grace_period: {{ . }} - {{- end }} - {{- if $service.privileged }} - privileged: true - {{- end }} - {{- with $service.pid }} - pid: {{ . }} - {{- end }} - {{- if ne (toString $service.noNewPrivileges) "false" }} - security_opt: - - no-new-privileges:true - {{- end }} - {{- if and $values.sleep.after (not $service.once) }} - labels: - sablier.enable: "true" - sablier.group: {{ $values.slug }} - {{- end }} - deploy: - resources: - limits: - cpus: {{ toString ($service.cpus | default $values.service.cpus) | quote }} - memory: {{ $memory }} - {{- if not $service.once }} - reservations: - memory: {{ $service.reserve | default (printf "%dM" (div (include "core.megabytes" $memory | atoi) 4)) }} - {{- end }} -{{- end }} -{{- if $volumes }} - -volumes: - {{- range $volume, $_ := $volumes }} - {{ $volume }}: {} - {{- end }} -{{- end }} -{{- if $configs }} - -configs: - {{- range $configs }} - {{ .name }}: - content: {{ replace "$" "$$" .content | toRawJson }} - {{- end }} -{{- end }} -{{- if or $public (include "core.hosted" . | fromJson).names }} - -networks: - edge: - external: true -{{- end }} diff --git a/charts/core/templates/route.yaml b/charts/core/templates/route.yaml deleted file mode 100644 index 918471f..0000000 --- a/charts/core/templates/route.yaml +++ /dev/null @@ -1,59 +0,0 @@ -{{- $values := .Values -}} -{{- $public := include "core.public" . -}} -{{- if $values.host }} -{{- if not $public }}{{ fail "no service gets the hostname: give one service a port, or set public: true on one of them" }}{{ end }} -{{- $services := include "core.services" . | fromJson }} -{{- $service := index $services $public }} -{{- if not $service.port }}{{ fail (printf "%s gets the hostname, so it must set port" $public) }}{{ end }} -{{- $hosted := (include "core.hosted" . | fromJson).names }} -http: - routers: - {{ $values.slug }}: - rule: {{ printf "Host(`%s`)" $values.host | quote }} - entryPoints: [websecure] - service: {{ $values.slug }} - tls: - certResolver: letsencrypt - {{- if $values.sleep.after }} - middlewares: [{{ $values.slug }}-sleep] - {{- end }} - {{- range $name := $hosted }} - {{- range $index, $host := (index $services $name).hosts }} - {{ printf "%s-%s-%d" $values.slug $name $index }}: - rule: {{ printf "Host(`%s`)" $host | quote }} - entryPoints: [websecure] - service: {{ printf "%s-%s" $values.slug $name }} - tls: - certResolver: letsencrypt - {{- if $values.sleep.after }} - middlewares: [{{ $values.slug }}-sleep] - {{- end }} - {{- end }} - {{- end }} - services: - {{ $values.slug }}: - loadBalancer: - servers: - - url: {{ printf "http://%s:%v" $values.slug $service.port | quote }} - {{- range $name := $hosted }} - {{ printf "%s-%s" $values.slug $name }}: - loadBalancer: - servers: - - url: {{ printf "http://%s-%s:%v" $values.slug $name (index $services $name).port | quote }} - {{- end }} - {{- if $values.sleep.after }} - middlewares: - {{ $values.slug }}-sleep: - plugin: - sablier: - sablierUrl: http://sablier:10000 - group: {{ $values.slug }} - sessionDuration: {{ $values.sleep.after }} - ignoreUserAgent: - - "(?i)(bot|crawler|spider|preview|curl|wget|uptime)" - dynamic: - displayName: {{ $values.name | default $values.slug | quote }} - theme: graphite - refreshFrequency: 5s - {{- end }} -{{- end }} diff --git a/charts/core/values.schema.json b/charts/core/values.schema.json deleted file mode 100644 index 5b4954e..0000000 --- a/charts/core/values.schema.json +++ /dev/null @@ -1,152 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Project Graphite project", - "type": "object", - "additionalProperties": false, - "required": ["core", "slug", "services"], - "properties": { - "core": {"type": "string", "pattern": "^[0-9]+[.][0-9]+[.][0-9]+$"}, - "slug": {"type": "string", "pattern": "^[a-z0-9](?:[a-z0-9-]{0,38}[a-z0-9])?$"}, - "name": {"type": "string"}, - "host": {"type": "string"}, - "tags": {"type": "object", "additionalProperties": {"type": "string"}}, - "enabled": {"type": "boolean"}, - "proxied": {"type": "boolean"}, - "sleep": { - "type": "object", - "additionalProperties": false, - "properties": { - "after": {"$ref": "#/definitions/duration"}, - "wake": {"$ref": "#/definitions/cron"}, - "wakeFor": {"$ref": "#/definitions/duration"} - }, - "dependencies": {"wake": ["after"], "wakeFor": ["wake"]} - }, - "service": { - "type": "object", - "additionalProperties": false, - "properties": {"memory": {"$ref": "#/definitions/memory"}, "cpus": {"$ref": "#/definitions/cpus"}} - }, - "postgres": { - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": {"type": "boolean"}, - "image": {"type": "string", "minLength": 1}, - "database": {"$ref": "#/definitions/identifier"}, - "user": {"$ref": "#/definitions/identifier"}, - "settings": {"type": "object", "additionalProperties": {"type": ["string", "number"]}}, - "env": {"$ref": "#/definitions/env"}, - "memory": {"$ref": "#/definitions/memory"}, - "cpus": {"$ref": "#/definitions/cpus"}, - "reserve": {"$ref": "#/definitions/optionalMemory"} - } - }, - "redis": { - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": {"type": "boolean"}, - "image": {"type": "string", "minLength": 1}, - "password": {"type": "boolean"}, - "evict": {"type": "boolean"}, - "maxmemory": {"type": "string", "pattern": "^([0-9]+(mb|gb))?$"}, - "memory": {"$ref": "#/definitions/memory"}, - "cpus": {"$ref": "#/definitions/cpus"}, - "reserve": {"$ref": "#/definitions/optionalMemory"} - } - }, - "backup": { - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": {"type": "boolean"}, - "keepDays": {"type": "integer", "minimum": 1}, - "include": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/definitions/volumeName"}}, - "memory": {"$ref": "#/definitions/memory"}, - "cpus": {"$ref": "#/definitions/cpus"}, - "reserve": {"$ref": "#/definitions/optionalMemory"} - } - }, - "services": { - "type": "object", - "propertyNames": {"pattern": "^[a-z][a-z0-9-]*$", "not": {"enum": ["postgres", "redis", "backup"]}}, - "additionalProperties": {"$ref": "#/definitions/service"} - } - }, - "definitions": { - "duration": {"type": "string", "pattern": "^[0-9]+(s|m|h)$"}, - "cron": {"type": "string", "pattern": "^(@every [0-9]+[smh]|@(hourly|daily|weekly|monthly)|[^ ]+( [^ ]+){4})$"}, - "hostname": {"type": "string", "pattern": "^([a-z0-9]([a-z0-9-]*[a-z0-9])?[.])+project-graphite[.]com$"}, - "memory": {"type": "string", "pattern": "^[0-9]+[MG]$"}, - "optionalMemory": {"type": "string", "pattern": "^([0-9]+[MG])?$"}, - "cpus": {"type": ["string", "number"]}, - "identifier": {"type": "string", "pattern": "^([a-z_][a-z0-9_]*)?$"}, - "volumeName": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$"}, - "strings": {"type": "array", "minItems": 1, "items": {"type": "string"}}, - "names": {"type": "array", "uniqueItems": true, "items": {"type": "string", "pattern": "^[A-Z][A-Z0-9_]*$"}}, - "env": {"type": "object", "additionalProperties": {"type": ["string", "number", "boolean"]}}, - "service": { - "type": "object", - "additionalProperties": false, - "required": ["image"], - "properties": { - "image": {"type": "string", "minLength": 1}, - "tag": {"type": "string", "minLength": 1}, - "entrypoint": {"$ref": "#/definitions/strings"}, - "command": {"$ref": "#/definitions/strings"}, - "port": {"type": "integer", "minimum": 1, "maximum": 65535}, - "public": {"type": "boolean"}, - "hosts": {"type": "array", "uniqueItems": true, "minItems": 1, "items": {"$ref": "#/definitions/hostname"}}, - "jobs": { - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "additionalProperties": false, - "required": ["name", "schedule", "command"], - "properties": { - "name": {"type": "string", "pattern": "^[a-z][a-z0-9-]*$"}, - "schedule": {"$ref": "#/definitions/cron"}, - "command": {"$ref": "#/definitions/strings"}, - "user": {"type": "string", "minLength": 1} - } - } - }, - "env": {"$ref": "#/definitions/env"}, - "secrets": {"$ref": "#/definitions/names"}, - "optional": {"$ref": "#/definitions/names"}, - "dependsOn": {"type": "object", "additionalProperties": {"enum": ["healthy", "started", "completed"]}}, - "volumes": {"type": "array", "uniqueItems": true, "items": {"type": "string", "pattern": "^[^:]+:/[^:]*(:ro)?$"}}, - "files": {"type": "object", "propertyNames": {"pattern": "^/"}, "additionalProperties": {"type": "string"}}, - "health": { - "oneOf": [ - {"type": "string", "pattern": "^/"}, - { - "type": "object", - "additionalProperties": false, - "properties": { - "path": {"type": "string", "pattern": "^/"}, - "test": {"$ref": "#/definitions/strings"}, - "interval": {"$ref": "#/definitions/duration"}, - "timeout": {"$ref": "#/definitions/duration"}, - "retries": {"type": "integer", "minimum": 1}, - "startPeriod": {"$ref": "#/definitions/duration"} - }, - "oneOf": [{"required": ["path"]}, {"required": ["test"]}] - } - ] - }, - "memory": {"$ref": "#/definitions/memory"}, - "cpus": {"$ref": "#/definitions/cpus"}, - "reserve": {"$ref": "#/definitions/memory"}, - "once": {"type": "boolean"}, - "privileged": {"type": "boolean"}, - "pid": {"enum": ["host"]}, - "init": {"type": "boolean"}, - "noNewPrivileges": {"type": "boolean"}, - "stopGracePeriod": {"$ref": "#/definitions/duration"} - } - } - } -} diff --git a/charts/core/values.yaml b/charts/core/values.yaml deleted file mode 100644 index 0c849f0..0000000 --- a/charts/core/values.yaml +++ /dev/null @@ -1,72 +0,0 @@ -# Every option a project can set, with its default. A project writes only what differs, in -# deploy//core.yaml (what runs) and deploy//values.yaml (its settings). - -# Filled in by Deploy from projects/.yml and tags/.json. Never write these. -slug: "" -name: "" -host: "" -tags: {} - -# The project. -enabled: true # false takes the project down -proxied: false # true sends the hostname through Cloudflare's proxy -sleep: {} # after: 30m stops the project after that long without requests and starts - # it on the next one; wake: "0 */6 * * *" (cron or "@every 6h") starts it on a - # schedule for wakeFor (default 15m) - -# Every service under services: starts from these. A service sets: -# image required; the tag comes from tags/.json unless the image names one or tag: is set -# port the port it listens on; with one port in the project, that service gets the hostname -# public true picks the service that gets the hostname when several have a port -# hosts more hostnames for this service, e.g. [api.orbit.project-graphite.com]; needs port -# jobs [{name: nightly, schedule: "0 3 * * *", command: [python, -m, reports]}] runs the -# command inside this running container on a cron schedule (UTC; user: to change -# the user, root by default); not for sleeping projects -# health an HTTP path on port, e.g. /health (checked with wget, then curl), or -# {test: [CMD, ...], interval: 30s, timeout: 5s, retries: 3, startPeriod: 0s} -# env plain settings; ${NAME} and ${NAME:-default} read the secrets file -# secrets names that must be in secrets/.env, passed through as environment -# optional names passed through when present, empty otherwise -# volumes [name:/path] or [name:/path:ro]; named volumes are created -# files {/path/in/container: file-in-deploy//files} -# dependsOn {service: healthy | started | completed}; services also wait for postgres and redis -# command, entrypoint, stopGracePeriod, once (runs to completion), privileged, pid: host, -# init (default true), noNewPrivileges (default true) -# memory, cpus, reserve (default a quarter of memory) -service: - memory: 256M - cpus: "0.5" - -# A PostgreSQL server named postgres, with its data in the postgres-data volume. Every service gets -# DATABASE_URL, POSTGRES_HOST, POSTGRES_PORT, POSTGRES_DB, POSTGRES_USER and POSTGRES_PASSWORD, and -# POSTGRES_PASSWORD must be in the secrets file. -postgres: - image: postgres:18-alpine - database: "" # default: the slug with - as _ - user: "" # default: the database name - settings: {} # server settings, e.g. {shared_buffers: 64MB, max_connections: 40} - env: {} - memory: 256M - cpus: "0.5" - reserve: "" - -# A Redis server named redis, append-only, with its data in the redis-data volume. Every service -# gets REDIS_URL. -redis: - image: redis:8.10-alpine - password: false # true requires REDIS_PASSWORD in the secrets file - evict: true # false keeps every key and refuses writes when full, as a queue needs - maxmemory: "" # default: three quarters of memory - memory: 64M - cpus: "0.25" - reserve: "" - -# A daily pg_dump of postgres into the backups volume, plus a tar of each volume in include. -backup: - keepDays: 14 - include: [] # volume names, e.g. [files-data] - memory: 128M - cpus: "0.25" - reserve: "" - -services: {} diff --git a/deploy/chart.yaml b/deploy/chart.yaml deleted file mode 100644 index 5e8c0e6..0000000 --- a/deploy/chart.yaml +++ /dev/null @@ -1 +0,0 @@ -core: 2.0.0 diff --git a/deploy/console/compose.yaml b/deploy/console/compose.yaml new file mode 100644 index 0000000..45a85a2 --- /dev/null +++ b/deploy/console/compose.yaml @@ -0,0 +1,26 @@ +services: + app: + extends: {file: ../../shared/app.yaml, service: app} + image: ghcr.io/project-graphite/console/app:${IMAGE_TAG:?} + environment: + NODE_ENV: production + SSH_TARGET: ubuntu@135.125.254.59 + GITHUB_CLIENT_ID: ${GITHUB_CLIENT_ID:?} + GITHUB_CLIENT_SECRET: ${GITHUB_CLIENT_SECRET:?} + SESSION_SECRET: ${SESSION_SECRET:?} + ACCESS_TEAM: ${ACCESS_TEAM:?} + ACCESS_AUD: ${ACCESS_AUD:?} + ALERTS_TOKEN: ${ALERTS_TOKEN:-} + SECRETS_KEY: ${SECRETS_KEY:-} + privileged: true + pid: host + volumes: [/var/run/docker.sock:/var/run/docker.sock, data:/data] + healthcheck: + test: [CMD, node, -e, "fetch('http://127.0.0.1:3000/healthz').then((r) => process.exit(r.ok ? 0 : 1), () => process.exit(1))"] + start_period: 10s + +volumes: + data: {labels: {graphite.backup: "true"}} + +networks: + edge: {external: true} diff --git a/deploy/console/core.yaml b/deploy/console/core.yaml deleted file mode 100644 index de9266a..0000000 --- a/deploy/console/core.yaml +++ /dev/null @@ -1,12 +0,0 @@ -proxied: true - -services: - app: - image: ghcr.io/project-graphite/console/app - port: 3000 - privileged: true - pid: host - volumes: [/var/run/docker.sock:/var/run/docker.sock, data:/data] - health: - test: [CMD, node, -e, "fetch('http://127.0.0.1:3000/healthz').then((r) => process.exit(r.ok ? 0 : 1), () => process.exit(1))"] - startPeriod: 10s diff --git a/deploy/console/values b/deploy/console/values new file mode 100644 index 0000000..1d57523 --- /dev/null +++ b/deploy/console/values @@ -0,0 +1,5 @@ +IMAGE_TAG=sha-8d9bb17 +PROXIED=true +APP_MEMORY=192M +APP_CPUS=0.25 +APP_RESERVE=48M diff --git a/deploy/console/values.yaml b/deploy/console/values.yaml deleted file mode 100644 index 2f9bf8d..0000000 --- a/deploy/console/values.yaml +++ /dev/null @@ -1,9 +0,0 @@ -app: - memory: 192M - cpus: "0.25" - env: - NODE_ENV: production - APP_URL: https://console.project-graphite.com - SSH_TARGET: ubuntu@135.125.254.59 - secrets: [GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET, SESSION_SECRET, ACCESS_TEAM, ACCESS_AUD] - optional: [ALERTS_TOKEN] diff --git a/deploy/digital-to-physical/values b/deploy/digital-to-physical/values new file mode 100644 index 0000000..5989882 --- /dev/null +++ b/deploy/digital-to-physical/values @@ -0,0 +1 @@ +IMAGE_TAG=sha-d5ba7b2 diff --git a/deploy/digital-to-physical/values.yaml b/deploy/digital-to-physical/values.yaml deleted file mode 100644 index d4ca941..0000000 --- a/deploy/digital-to-physical/values.yaml +++ /dev/null @@ -1 +0,0 @@ -enabled: true diff --git a/deploy/graphite-hub/values b/deploy/graphite-hub/values new file mode 100644 index 0000000..2b0efa4 --- /dev/null +++ b/deploy/graphite-hub/values @@ -0,0 +1 @@ +IMAGE_TAG=sha-ff5f3e8 diff --git a/deploy/graphite-hub/values.yaml b/deploy/graphite-hub/values.yaml deleted file mode 100644 index d4ca941..0000000 --- a/deploy/graphite-hub/values.yaml +++ /dev/null @@ -1 +0,0 @@ -enabled: true diff --git a/deploy/graphite-message-lab/core.yaml b/deploy/graphite-message-lab/core.yaml deleted file mode 100644 index a7a28fd..0000000 --- a/deploy/graphite-message-lab/core.yaml +++ /dev/null @@ -1,26 +0,0 @@ -sleep: - after: 30m - -postgres: - image: postgres:18.6-trixie -redis: - image: redis:8.10.2-trixie - evict: false - -services: - web: - image: ghcr.io/project-graphite/graphite-message-lab/web - command: [sh, -c, "python manage.py migrate --noinput && exec gunicorn -c python:config.gunicorn config.wsgi"] - port: 8000 - health: {test: [CMD, curl, --fail, "http://localhost:8000/up/databases"], startPeriod: 20s} - stopGracePeriod: 10s - - worker: - image: ghcr.io/project-graphite/graphite-message-lab/worker - command: [celery, -A, config, worker, --loglevel=info, --concurrency=1, --max-tasks-per-child=100] - health: - test: [CMD-SHELL, "celery -A config inspect ping -d celery@$$HOSTNAME -t 10"] - interval: 60s - timeout: 30s - startPeriod: 30s - stopGracePeriod: 30s diff --git a/deploy/graphite-message-lab/values.yaml b/deploy/graphite-message-lab/values.yaml deleted file mode 100644 index c21dbed..0000000 --- a/deploy/graphite-message-lab/values.yaml +++ /dev/null @@ -1,18 +0,0 @@ -web: - memory: 512M - env: &settings - ALLOWED_HOSTS: message-lab.project-graphite.com,localhost - CSRF_TRUSTED_ORIGINS: https://message-lab.project-graphite.com - WEB_CONCURRENCY: "1" - secrets: [SECRET_KEY] - -worker: - memory: 384M - env: *settings - secrets: [SECRET_KEY] - -postgres: - memory: 512M - -redis: - memory: 128M diff --git a/deploy/graphite-tracker/compose.yaml b/deploy/graphite-tracker/compose.yaml new file mode 100644 index 0000000..ce92a27 --- /dev/null +++ b/deploy/graphite-tracker/compose.yaml @@ -0,0 +1,47 @@ +services: + app: + extends: {file: ../../shared/app.yaml, service: app} + image: ghcr.io/project-graphite/graphite-tracker/app:${IMAGE_TAG:?} + environment: + AUTH_TRUSTED_ORIGINS: https://${HOST} + DATABASE_URL: postgresql://graphite:${POSTGRES_PASSWORD:?}@postgres:5432/graphite_tracker + POSTGRES_HOST: postgres + POSTGRES_PORT: "5432" + POSTGRES_DB: graphite_tracker + POSTGRES_USER: graphite + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?} + REDIS_URL: redis://redis:6379 + AUTH_ACCESS_TOKEN_SECRET: ${AUTH_ACCESS_TOKEN_SECRET:?} + GAME_SOURCE: ${GAME_SOURCE:-igdb} + DISABLED_SOURCES: ${DISABLED_SOURCES:-} + TMDB_READ_ACCESS_TOKEN: ${TMDB_READ_ACCESS_TOKEN:-} + IGDB_CLIENT_ID: ${IGDB_CLIENT_ID:-} + IGDB_CLIENT_SECRET: ${IGDB_CLIENT_SECRET:-} + RAWG_API_KEY: ${RAWG_API_KEY:-} + EMAIL_HOST: ${EMAIL_HOST:?} + EMAIL_PORT: ${EMAIL_PORT:-587} + DEFAULT_FROM_EMAIL: ${DEFAULT_FROM_EMAIL:?} + EMAIL_HOST_USER: ${EMAIL_HOST_USER:-} + EMAIL_HOST_PASSWORD: ${EMAIL_HOST_PASSWORD:-} + healthcheck: + test: [CMD-SHELL, "wget -q -O /dev/null http://127.0.0.1:3000/api/v1/health || curl -fsS -o /dev/null http://127.0.0.1:3000/api/v1/health"] + start_period: 20s + depends_on: + postgres: {condition: service_healthy} + redis: {condition: service_healthy} + + postgres: + extends: {file: ../../shared/postgres.yaml, service: postgres} + environment: + POSTGRES_DB: graphite_tracker + POSTGRES_USER: graphite + + redis: + extends: {file: ../../shared/redis.yaml, service: redis} + +volumes: + postgres-data: {} + redis-data: {} + +networks: + edge: {external: true} diff --git a/deploy/graphite-tracker/core.yaml b/deploy/graphite-tracker/core.yaml deleted file mode 100644 index 8ec8645..0000000 --- a/deploy/graphite-tracker/core.yaml +++ /dev/null @@ -1,10 +0,0 @@ -postgres: - database: graphite_tracker - user: graphite -redis: {} - -services: - app: - image: ghcr.io/project-graphite/graphite-tracker/app - port: 3000 - health: {path: /api/v1/health, startPeriod: 20s} diff --git a/deploy/graphite-tracker/values b/deploy/graphite-tracker/values new file mode 100644 index 0000000..adf12cf --- /dev/null +++ b/deploy/graphite-tracker/values @@ -0,0 +1,10 @@ +IMAGE_TAG=sha-7533f37 +APP_MEMORY=512M +APP_CPUS=0.75 +APP_RESERVE=128M +POSTGRES_MEMORY=384M +POSTGRES_CPUS=0.75 +POSTGRES_RESERVE=96M +REDIS_MEMORY=96M +REDIS_RESERVE=24M +REDIS_MAXMEMORY=64mb diff --git a/deploy/graphite-tracker/values.yaml b/deploy/graphite-tracker/values.yaml deleted file mode 100644 index 313e9d1..0000000 --- a/deploy/graphite-tracker/values.yaml +++ /dev/null @@ -1,18 +0,0 @@ -app: - memory: 512M - cpus: "0.75" - env: - APP_URL: https://graphite-tracker.project-graphite.com - AUTH_TRUSTED_ORIGINS: https://graphite-tracker.project-graphite.com - GAME_SOURCE: ${GAME_SOURCE:-igdb} - EMAIL_PORT: ${EMAIL_PORT:-587} - secrets: [AUTH_ACCESS_TOKEN_SECRET, EMAIL_HOST, DEFAULT_FROM_EMAIL] - optional: [TMDB_READ_ACCESS_TOKEN, IGDB_CLIENT_ID, IGDB_CLIENT_SECRET, RAWG_API_KEY, DISABLED_SOURCES, EMAIL_HOST_USER, EMAIL_HOST_PASSWORD] - -postgres: - memory: 384M - cpus: "0.75" - -redis: - memory: 96M - maxmemory: 64mb diff --git a/deploy/pixelpocket/compose.yaml b/deploy/pixelpocket/compose.yaml new file mode 100644 index 0000000..f90876c --- /dev/null +++ b/deploy/pixelpocket/compose.yaml @@ -0,0 +1,100 @@ +services: + frontend: + extends: {file: ../../shared/app.yaml, service: app} + image: ghcr.io/project-graphite/pixelpocket/frontend:${IMAGE_TAG:?} + healthcheck: + test: [CMD-SHELL, "wget -q -O /dev/null http://127.0.0.1:80/ || curl -fsS -o /dev/null http://127.0.0.1:80/"] + start_period: 10s + depends_on: + backend: {condition: service_healthy} + + backend: + extends: {file: ../../shared/app.yaml, service: service} + image: ghcr.io/project-graphite/pixelpocket/backend:${IMAGE_TAG:?} + expose: ["5000"] + environment: + NODE_ENV: production + PORT: "5000" + DATABASE_LOCAL: mongodb://pixelpocket:${MONGO_PASSWORD:?}@mongo:27017/pixelpocket?authSource=admin + JWT_SECRET: ${JWT_SECRET:?} + JWT_EXPIRES_IN: ${JWT_EXPIRES_IN:-90d} + JWT_COOKIE_EXPIRES_IN: ${JWT_COOKIE_EXPIRES_IN:-90} + FRONTEND_URL: https://${HOST} + EMAIL_HOST: ${EMAIL_HOST:?} + EMAIL_PORT: ${EMAIL_PORT:-587} + EMAIL_FROM: ${EMAIL_FROM:?} + EMAIL_USERNAME: ${EMAIL_USERNAME:-} + EMAIL_PASSWORD: ${EMAIL_PASSWORD:-} + S3_ENDPOINT: http://minio:9000 + S3_BUCKET: pixelpocket + S3_ACCESS_KEY: pixelpocket-app + S3_SECRET_KEY: ${S3_SECRET_KEY:?} + healthcheck: + test: [CMD-SHELL, "wget -q -O /dev/null http://127.0.0.1:5000/api/v1/users/session || curl -fsS -o /dev/null http://127.0.0.1:5000/api/v1/users/session"] + start_period: 20s + depends_on: + mongo: {condition: service_healthy} + minio-init: {condition: service_completed_successfully} + + mongo: + extends: {file: ../../shared/app.yaml, service: service} + image: mongo:7 + environment: + MONGO_INITDB_ROOT_USERNAME: pixelpocket + MONGO_INITDB_ROOT_PASSWORD: ${MONGO_PASSWORD:?} + volumes: [mongo-data:/data/db] + healthcheck: + test: [CMD-SHELL, "mongosh -u \"$$MONGO_INITDB_ROOT_USERNAME\" -p \"$$MONGO_INITDB_ROOT_PASSWORD\" --authenticationDatabase admin --quiet --eval \"db.adminCommand('ping')\""] + interval: 10s + retries: 10 + deploy: + resources: + limits: {memory: 512M} + reservations: {memory: 128M} + + minio: + extends: {file: ../../shared/app.yaml, service: service} + image: chainguard/minio@sha256:25bc63a89475bc494bfbce9bbeddb0421aece3f3968d77c809db5104d07b4545 + command: [server, /data] + environment: + MINIO_ROOT_USER: pixelpocket-admin + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?} + volumes: [minio-data:/data] + configs: + - {source: minio-healthcheck, target: /scripts/healthcheck.sh} + healthcheck: + test: [CMD, bash, /scripts/healthcheck.sh] + interval: 10s + retries: 10 + + minio-init: + extends: {file: ../../shared/app.yaml, service: service} + image: chainguard/minio@sha256:25bc63a89475bc494bfbce9bbeddb0421aece3f3968d77c809db5104d07b4545 + entrypoint: [bash, /scripts/init.sh] + restart: "no" + environment: + MINIO_ROOT_USER: pixelpocket-admin + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?} + S3_BUCKET: pixelpocket + S3_ACCESS_KEY: pixelpocket-app + S3_SECRET_KEY: ${S3_SECRET_KEY:?} + configs: + - {source: minio-init, target: /scripts/init.sh} + healthcheck: !reset {} + depends_on: + minio: {condition: service_healthy} + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + reservations: !reset {} + +volumes: + mongo-data: {} + minio-data: {} + +configs: + minio-healthcheck: {file: ./files/minio-healthcheck.sh} + minio-init: {file: ./files/minio-init.sh} + +networks: + edge: {external: true} diff --git a/deploy/pixelpocket/core.yaml b/deploy/pixelpocket/core.yaml deleted file mode 100644 index 619204a..0000000 --- a/deploy/pixelpocket/core.yaml +++ /dev/null @@ -1,35 +0,0 @@ -services: - frontend: - image: ghcr.io/project-graphite/pixelpocket/frontend - port: 80 - public: true - dependsOn: {backend: healthy} - health: {path: /, startPeriod: 10s} - - backend: - image: ghcr.io/project-graphite/pixelpocket/backend - port: 5000 - dependsOn: {mongo: healthy, minio-init: completed} - health: {path: /api/v1/users/session, startPeriod: 20s} - - mongo: - image: mongo:7 - volumes: [mongo-data:/data/db] - health: - test: [CMD-SHELL, "mongosh -u \"$$MONGO_INITDB_ROOT_USERNAME\" -p \"$$MONGO_INITDB_ROOT_PASSWORD\" --authenticationDatabase admin --quiet --eval \"db.adminCommand('ping')\""] - interval: 10s - retries: 10 - - minio: - image: chainguard/minio@sha256:25bc63a89475bc494bfbce9bbeddb0421aece3f3968d77c809db5104d07b4545 - command: [server, /data] - volumes: [minio-data:/data] - files: {/scripts/healthcheck.sh: minio-healthcheck.sh} - health: {test: [CMD, bash, /scripts/healthcheck.sh], interval: 10s, retries: 10} - - minio-init: - image: chainguard/minio@sha256:25bc63a89475bc494bfbce9bbeddb0421aece3f3968d77c809db5104d07b4545 - entrypoint: [bash, /scripts/init.sh] - once: true - dependsOn: {minio: healthy} - files: {/scripts/init.sh: minio-init.sh} diff --git a/deploy/pixelpocket/values b/deploy/pixelpocket/values new file mode 100644 index 0000000..1a94ea6 --- /dev/null +++ b/deploy/pixelpocket/values @@ -0,0 +1,5 @@ +IMAGE_TAG=sha-e38c8e1 +APP_PORT=80 +APP_MEMORY=64M +APP_CPUS=0.25 +APP_RESERVE=16M diff --git a/deploy/pixelpocket/values.yaml b/deploy/pixelpocket/values.yaml deleted file mode 100644 index f05343e..0000000 --- a/deploy/pixelpocket/values.yaml +++ /dev/null @@ -1,41 +0,0 @@ -frontend: - memory: 64M - cpus: "0.25" - -backend: - memory: 256M - env: - NODE_ENV: production - PORT: "5000" - DATABASE_LOCAL: mongodb://pixelpocket:${MONGO_PASSWORD}@mongo:27017/pixelpocket?authSource=admin - JWT_EXPIRES_IN: ${JWT_EXPIRES_IN:-90d} - JWT_COOKIE_EXPIRES_IN: ${JWT_COOKIE_EXPIRES_IN:-90} - FRONTEND_URL: https://pixelpocket.project-graphite.com - EMAIL_PORT: ${EMAIL_PORT:-587} - S3_ENDPOINT: http://minio:9000 - S3_BUCKET: pixelpocket - S3_ACCESS_KEY: pixelpocket-app - secrets: [JWT_SECRET, EMAIL_HOST, EMAIL_FROM, S3_SECRET_KEY] - optional: [EMAIL_USERNAME, EMAIL_PASSWORD] - -mongo: - memory: 512M - env: - MONGO_INITDB_ROOT_USERNAME: pixelpocket - MONGO_INITDB_ROOT_PASSWORD: ${MONGO_PASSWORD} - secrets: [MONGO_PASSWORD] - -minio: - memory: 256M - env: - MINIO_ROOT_USER: pixelpocket-admin - secrets: [MINIO_ROOT_PASSWORD] - -minio-init: - memory: 128M - cpus: "0.25" - env: - MINIO_ROOT_USER: pixelpocket-admin - S3_BUCKET: pixelpocket - S3_ACCESS_KEY: pixelpocket-app - secrets: [MINIO_ROOT_PASSWORD, S3_SECRET_KEY] diff --git a/deploy/strata/compose.yaml b/deploy/strata/compose.yaml new file mode 100644 index 0000000..d53bdd0 --- /dev/null +++ b/deploy/strata/compose.yaml @@ -0,0 +1,45 @@ +services: + app: + extends: {file: ../../shared/app.yaml, service: app} + image: ghcr.io/project-graphite/strata/app:${IMAGE_TAG:?} + environment: + NODE_ENV: production + AUTH_TRUSTED_ORIGINS: https://${HOST} + DATABASE_URL: postgresql://${PROJECT}:${POSTGRES_PASSWORD:?}@postgres:5432/${PROJECT} + POSTGRES_HOST: postgres + POSTGRES_PORT: "5432" + POSTGRES_DB: ${PROJECT} + POSTGRES_USER: ${PROJECT} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?} + REDIS_URL: redis://:${REDIS_PASSWORD:?}@redis:6379 + AUTH_ACCESS_TOKEN_SECRET: ${AUTH_ACCESS_TOKEN_SECRET:?} + DATA_ENCRYPTION_KEY: ${DATA_ENCRYPTION_KEY:?} + EMAIL_HOST: ${EMAIL_HOST:?} + EMAIL_PORT: ${EMAIL_PORT:-587} + DEFAULT_FROM_EMAIL: ${DEFAULT_FROM_EMAIL:?} + EMAIL_HOST_USER: ${EMAIL_HOST_USER:-} + EMAIL_HOST_PASSWORD: ${EMAIL_HOST_PASSWORD:-} + VAPID_PUBLIC_KEY: ${VAPID_PUBLIC_KEY:-} + VAPID_PRIVATE_KEY: ${VAPID_PRIVATE_KEY:-} + VAPID_SUBJECT: ${VAPID_SUBJECT:-} + volumes: [files-data:/data/files] + healthcheck: + test: [CMD-SHELL, "wget -q -O /dev/null http://127.0.0.1:3000/api/v1/health || curl -fsS -o /dev/null http://127.0.0.1:3000/api/v1/health"] + start_period: 20s + depends_on: + postgres: {condition: service_healthy} + redis: {condition: service_healthy} + + postgres: + extends: {file: ../../shared/postgres.yaml, service: postgres} + + redis: + extends: {file: ../../shared/redis-password.yaml, service: redis} + +volumes: + files-data: {labels: {graphite.backup: "true"}} + postgres-data: {} + redis-data: {} + +networks: + edge: {external: true} diff --git a/deploy/strata/core.yaml b/deploy/strata/core.yaml deleted file mode 100644 index d78c6c6..0000000 --- a/deploy/strata/core.yaml +++ /dev/null @@ -1,13 +0,0 @@ -postgres: - settings: {shared_buffers: 64MB, work_mem: 4MB, max_connections: 40} -redis: - password: true -backup: - include: [files-data] - -services: - app: - image: ghcr.io/project-graphite/strata/app - port: 3000 - health: {path: /api/v1/health, startPeriod: 20s} - volumes: [files-data:/data/files] diff --git a/deploy/strata/values b/deploy/strata/values new file mode 100644 index 0000000..706aa8c --- /dev/null +++ b/deploy/strata/values @@ -0,0 +1,8 @@ +IMAGE_TAG=sha-ebcde55 +APP_MEMORY=512M +APP_CPUS=0.75 +APP_RESERVE=128M +POSTGRES_MEMORY=320M +POSTGRES_RESERVE=80M +POSTGRES_SHARED_BUFFERS=64MB +POSTGRES_MAX_CONNECTIONS=40 diff --git a/deploy/strata/values.yaml b/deploy/strata/values.yaml deleted file mode 100644 index ad03597..0000000 --- a/deploy/strata/values.yaml +++ /dev/null @@ -1,13 +0,0 @@ -app: - memory: 512M - cpus: "0.75" - env: - NODE_ENV: production - APP_URL: https://strata.project-graphite.com - AUTH_TRUSTED_ORIGINS: https://strata.project-graphite.com - EMAIL_PORT: ${EMAIL_PORT:-587} - secrets: [AUTH_ACCESS_TOKEN_SECRET, DATA_ENCRYPTION_KEY, EMAIL_HOST, DEFAULT_FROM_EMAIL] - optional: [EMAIL_HOST_USER, EMAIL_HOST_PASSWORD, VAPID_PUBLIC_KEY, VAPID_PRIVATE_KEY, VAPID_SUBJECT] - -postgres: - memory: 320M diff --git a/deploy/traceboard/values.yaml b/deploy/traceboard/values.yaml deleted file mode 100644 index bc11441..0000000 --- a/deploy/traceboard/values.yaml +++ /dev/null @@ -1 +0,0 @@ -enabled: false diff --git a/edge/backup.py b/edge/backup.py new file mode 100644 index 0000000..e67126d --- /dev/null +++ b/edge/backup.py @@ -0,0 +1,403 @@ +import contextlib +import datetime +import hashlib +import http.client +import io +import json +import os +import pathlib +import shutil +import socket +import sqlite3 +import sys +import tarfile +import tempfile +import time +import traceback +import urllib.parse + +ROOT = pathlib.Path(os.environ.get("BACKUP_ROOT", "/backups")) +VOLUMES = pathlib.Path(os.environ.get("VOLUMES_ROOT", "/volumes")) +REQUESTS = ROOT / ".requests" +WORK = ROOT / ".work" +KEEP_DAYS = int(os.environ.get("KEEP_DAYS", "14")) +HOUR = int(os.environ.get("BACKUP_HOUR", "3")) +STAMP = "%Y%m%dT%H%M%SZ" +SQLITE = b"SQLite format 3\x00" +SIDECARS = ("-journal", "-wal", "-shm") +DUMP = "/tmp/graphite-backup.dump" + + +def log(message): + print(f"{datetime.datetime.now(datetime.UTC):%Y-%m-%dT%H:%M:%SZ} {message}", flush=True) + + +class Docker(http.client.HTTPConnection): + def __init__(self): + super().__init__("localhost", timeout=1800) + + def connect(self): + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.sock.settimeout(self.timeout) + self.sock.connect("/var/run/docker.sock") + + +def docker(method, path, body=None, stream=False, headers=None): + connection = Docker() + if isinstance(body, (dict, list)): + body, headers = json.dumps(body), {"Content-Type": "application/json"} + connection.request(method, path, body=body, headers=headers or {}) + response = connection.getresponse() + if response.status >= 400: + raise RuntimeError(f"Docker answered {response.status} to {method} {path.split('?')[0]}: {response.read().decode(errors='replace')[:300]}") + if stream: + return response + data = response.read() + connection.close() + return json.loads(data) if data else None + + +def filtered(path, **filters): + return f"{path}?all=1&filters={urllib.parse.quote(json.dumps(filters))}" + + +def execute(container, command): + created = docker("POST", f"/containers/{container}/exec", {"Cmd": command, "AttachStdout": True, "AttachStderr": True}) + response = docker("POST", f"/exec/{created['Id']}/start", {"Detach": False, "Tty": False}, stream=True) + output = {1: bytearray(), 2: bytearray()} + while header := response.read(8): + output.setdefault(header[0], bytearray()).extend(response.read(int.from_bytes(header[4:], "big"))) + code = docker("GET", f"/exec/{created['Id']}/json")["ExitCode"] + if code: + raise RuntimeError(f"{command[0]} exited with {code}: {output[2].decode(errors='replace')[-500:].strip()}") + return output[1].decode() + + +def digest(stream): + hasher = hashlib.sha256() + while chunk := stream.read(1 << 20): + hasher.update(chunk) + return hasher.hexdigest() + + +def file_digest(path): + with open(path, "rb") as stream: + return digest(stream) + + +def is_sqlite(path): + with open(path, "rb") as stream: + return stream.read(len(SQLITE)) == SQLITE + + +def copy_sqlite(source, target): + target.unlink(missing_ok=True) + with contextlib.closing(sqlite3.connect(f"{source.as_uri()}?mode=ro", uri=True)) as reader: + with contextlib.closing(sqlite3.connect(target)) as writer: + reader.backup(writer) + + +def integrity(path): + with contextlib.closing(sqlite3.connect(f"{path.as_uri()}?mode=ro", uri=True)) as database: + result = database.execute("PRAGMA integrity_check").fetchone()[0] + if result != "ok": + raise RuntimeError(f"integrity_check answered {result}") + + +def sidecar(path): + for suffix in SIDECARS: + if path.name.endswith(suffix): + base = path.with_name(path.name[: -len(suffix)]) + return base.is_file() and is_sqlite(base) + return False + + +def pack_volume(source, target, work): + databases = [] + with tarfile.open(target, "w:gz") as tar: + for path in sorted(source.rglob("*")): + name = path.relative_to(source).as_posix() + if path.is_file() and not path.is_symlink() and not sidecar(path) and is_sqlite(path): + copy = work / "copy.db" + copy_sqlite(path, copy) + info = tar.gettarinfo(path, name) + info.size = copy.stat().st_size + with open(copy, "rb") as stream: + tar.addfile(info, stream) + databases.append(name) + elif not (path.is_file() and sidecar(path)): + tar.add(path, name, recursive=False) + return databases + + +def check_volume(packed, item, work): + with tarfile.open(fileobj=packed, mode="r|gz") as inner: + for entry in inner: + if not entry.isfile(): + continue + with inner.extractfile(entry) as data: + if entry.name not in item["sqlite"]: + digest(data) + continue + copy = work / "check.db" + with open(copy, "wb") as out: + shutil.copyfileobj(data, out) + integrity(copy) + + +def verify(archive, project): + errors = [] + with tempfile.TemporaryDirectory(dir=WORK) as work, tarfile.open(archive) as tar: + manifest = json.load(tar.extractfile("manifest.json")) + if manifest.get("project") != project: + return [f"the archive is for {manifest.get('project')}, not {project}"] + for item in manifest["items"]: + try: + with tar.extractfile(item["path"]) as stream: + if digest(stream) != item["sha256"]: + errors.append(f"{item['path']} does not match its checksum") + continue + if item["kind"] == "volume": + with tar.extractfile(item["path"]) as packed: + check_volume(packed, item, pathlib.Path(work)) + except (KeyError, OSError, tarfile.TarError, sqlite3.Error, RuntimeError) as error: + errors.append(f"{item['path']}: {error}") + return errors + + +def dump_postgres(container, target): + if container["State"] != "running": + raise RuntimeError("its container is not running") + script = f'set -e; pg_dump -Fc -U "$POSTGRES_USER" -d "$POSTGRES_DB" -f {DUMP}; pg_restore --list {DUMP} > /dev/null; sha256sum {DUMP}' + try: + expected = execute(container["Id"], ["sh", "-c", script]).split()[0] + with tarfile.open(fileobj=docker("GET", f"/containers/{container['Id']}/archive?path={DUMP}", stream=True), mode="r|") as stream: + for member in stream: + with stream.extractfile(member) as source, open(target, "wb") as out: + shutil.copyfileobj(source, out) + finally: + execute(container["Id"], ["rm", "-f", DUMP]) + if file_digest(target) != expected: + raise RuntimeError("the dump changed on its way out of the container") + + +def targets(): + found = {} + for container in docker("GET", filtered("/containers/json", label=["graphite.backup.postgres=true"])): + project = container["Labels"].get("com.docker.compose.project", "other") + found.setdefault(project, {"postgres": [], "volumes": []})["postgres"].append(container) + for volume in docker("GET", filtered("/volumes", label=["graphite.backup=true"]))["Volumes"] or []: + project = (volume.get("Labels") or {}).get("com.docker.compose.project", "other") + found.setdefault(project, {"postgres": [], "volumes": []})["volumes"].append(volume) + return found + + +def write_summary(folder, stamp, summary): + temporary = folder / f"{stamp}.json.partial" + temporary.write_text(json.dumps(summary, indent=2), encoding="utf-8") + temporary.replace(folder / f"{stamp}.json") + + +def summarise(archive, project, manifest, errors): + errors = errors + verify(archive, project) + return {**manifest, "archive": archive.name, "bytes": archive.stat().st_size, "sha256": file_digest(archive), "ok": not errors, "errors": errors} + + +def backup(project, found, now): + stamp = now.strftime(STAMP) + folder = ROOT / project + folder.mkdir(parents=True, exist_ok=True) + archive = folder / f"{stamp}.tar" + manifest = {"project": project, "stamp": stamp, "started": now.isoformat(), "items": []} + errors = [] + with tempfile.TemporaryDirectory(dir=WORK) as work, tarfile.open(f"{archive}.partial", "w") as tar: + work = pathlib.Path(work) + + def add(path, name, **item): + tar.add(path, name) + manifest["items"].append({"path": name, "bytes": path.stat().st_size, "sha256": file_digest(path), **item}) + + for container in found["postgres"]: + service = container["Labels"].get("com.docker.compose.service", container["Id"][:12]) + try: + dump_postgres(container, work / f"{service}.dump") + add(work / f"{service}.dump", f"postgres/{service}.dump", kind="postgres", service=service) + except (RuntimeError, OSError, tarfile.TarError) as error: + errors.append(f"postgres {service}: {error}") + for volume in found["volumes"]: + name = (volume.get("Labels") or {}).get("com.docker.compose.volume", volume["Name"]) + try: + databases = pack_volume(VOLUMES / volume["Name"] / "_data", work / f"{name}.tar.gz", work) + add(work / f"{name}.tar.gz", f"volumes/{name}.tar.gz", kind="volume", volume=volume["Name"], sqlite=databases) + except (OSError, sqlite3.Error, tarfile.TarError) as error: + errors.append(f"volume {name}: {error}") + manifest["finished"] = datetime.datetime.now(datetime.UTC).isoformat() + data = json.dumps(manifest, indent=2).encode() + info = tarfile.TarInfo("manifest.json") + info.size, info.mtime = len(data), int(time.time()) + tar.addfile(info, io.BytesIO(data)) + pathlib.Path(f"{archive}.partial").replace(archive) + summary = summarise(archive, project, manifest, errors) + write_summary(folder, stamp, summary) + log(f"{project}: {archive.name}, {summary['bytes']} bytes, {'verified' if summary['ok'] else 'FAILED: ' + '; '.join(summary['errors'])}") + return summary + + +def prune(now): + cutoff = now - datetime.timedelta(days=KEEP_DAYS) + for path in ROOT.glob("*/*.tar"): + try: + stamp = datetime.datetime.strptime(path.stem, STAMP).replace(tzinfo=datetime.UTC) + except ValueError: + continue + if stamp < cutoff: + path.unlink() + path.with_suffix(".json").unlink(missing_ok=True) + log(f"removed {path.parent.name}/{path.name}") + for path in REQUESTS.glob("*.result"): + if path.stat().st_mtime < cutoff.timestamp(): + path.unlink() + + +def run(only=None): + now = datetime.datetime.now(datetime.UTC) + found = targets() + if only is not None and only not in found: + raise RuntimeError(f"{only} has no container labelled graphite.backup.postgres and no volume labelled graphite.backup") + summaries = [] + for project, items in sorted(found.items()): + if only is not None and project != only: + continue + try: + summaries.append(backup(project, items, now)) + except (OSError, RuntimeError, tarfile.TarError, json.JSONDecodeError) as error: + (ROOT / project).mkdir(parents=True, exist_ok=True) + write_summary(ROOT / project, now.strftime(STAMP), {"project": project, "stamp": now.strftime(STAMP), "started": now.isoformat(), "items": [], "ok": False, "errors": [str(error)]}) + log(f"{project}: FAILED: {error}") + summaries.append({"project": project, "ok": False, "errors": [str(error)]}) + prune(now) + return summaries + + +def project_containers(project): + return docker("GET", filtered("/containers/json", label=[f"com.docker.compose.project={project}"])) + + +def put_file(container, source, size, name, work): + bundle = work / "upload.tar" + with tarfile.open(bundle, "w") as tar: + info = tarfile.TarInfo(name) + info.size, info.mtime = size, int(time.time()) + tar.addfile(info, source) + with open(bundle, "rb") as body: + docker("PUT", f"/containers/{container}/archive?path=/tmp", body=body, headers={"Content-Type": "application/x-tar", "Content-Length": str(bundle.stat().st_size)}) + + +def wait_healthy(ids, timeout=300): + deadline = time.time() + timeout + while time.time() < deadline: + states = [docker("GET", f"/containers/{container}/json")["State"] for container in ids] + if all(state["Running"] and state.get("Health", {}).get("Status", "healthy") == "healthy" for state in states): + return + time.sleep(5) + raise RuntimeError("the project was not healthy five minutes after the restore") + + +def restore(project, name): + archive = ROOT / project / name + errors = verify(archive, project) + if errors: + raise RuntimeError("the archive failed verification: " + "; ".join(errors)) + containers = project_containers(project) + databases = {c["Labels"].get("com.docker.compose.service"): c for c in containers if c["Labels"].get("graphite.backup.postgres") == "true"} + running = [c["Id"] for c in containers if c["State"] == "running" and c["Id"] not in {d["Id"] for d in databases.values()}] + with tempfile.TemporaryDirectory(dir=WORK) as work, tarfile.open(archive) as tar: + work = pathlib.Path(work) + manifest = json.load(tar.extractfile("manifest.json")) + dumps = [item for item in manifest["items"] if item["kind"] == "postgres"] + for item in dumps: + database = databases.get(item["service"]) + if not database or database["State"] != "running": + raise RuntimeError(f"{project} has no running {item['service']} to restore into") + with tar.extractfile(item["path"]) as source: + put_file(database["Id"], source, item["bytes"], "graphite-restore.dump", work) + execute(database["Id"], ["pg_restore", "--list", "/tmp/graphite-restore.dump"]) + for container in running: + docker("POST", f"/containers/{container}/stop?t=30") + try: + for item in dumps: + execute(databases[item["service"]]["Id"], [ + "sh", "-c", + 'pg_restore --clean --if-exists --no-owner --single-transaction -U "$POSTGRES_USER" -d "$POSTGRES_DB" /tmp/graphite-restore.dump', + ]) + for item in (entry for entry in manifest["items"] if entry["kind"] == "volume"): + target = VOLUMES / item["volume"] / "_data" + for child in target.iterdir(): + if child.is_dir() and not child.is_symlink(): + shutil.rmtree(child) + else: + child.unlink() + with tar.extractfile(item["path"]) as packed, tarfile.open(fileobj=packed, mode="r|gz") as inner: + inner.extractall(target, filter="tar") + finally: + for database in databases.values(): + execute(database["Id"], ["rm", "-f", "/tmp/graphite-restore.dump"]) + for container in running: + docker("POST", f"/containers/{container}/start") + wait_healthy([*running, *(d["Id"] for d in databases.values())]) + return f"Restored {project} from {name}: {len(dumps)} databases and {len(manifest['items']) - len(dumps)} volumes; it is healthy." + + +def handle(request): + working = request.with_suffix(".running") + request.replace(working) + order = json.loads(working.read_text(encoding="utf-8")) + log(f"{order['action']} {order.get('project')} {order.get('archive', '')} for {order.get('by')}") + result = {**order, "at": datetime.datetime.now(datetime.UTC).isoformat()} + try: + if order["action"] == "run": + summaries = run(order["project"]) + result.update(ok=all(entry["ok"] for entry in summaries), output="; ".join(f"{entry['project']}: {'verified' if entry['ok'] else '; '.join(entry['errors'])}" for entry in summaries)) + elif order["action"] == "verify": + archive = ROOT / order["project"] / order["archive"] + with tarfile.open(archive) as tar: + manifest = json.load(tar.extractfile("manifest.json")) + summary = summarise(archive, order["project"], {**manifest, "stamp": archive.stem, "imported": True}, []) + write_summary(archive.parent, archive.stem, summary) + result.update(ok=summary["ok"], output="verified" if summary["ok"] else "; ".join(summary["errors"])) + elif order["action"] == "restore": + result.update(ok=True, output=restore(order["project"], order["archive"])) + else: + result.update(ok=False, output=f"unknown action {order['action']}") + except Exception as error: + traceback.print_exc() + result.update(ok=False, output=str(error)) + working.with_suffix(".result").write_text(json.dumps(result), encoding="utf-8") + working.unlink() + log(f"{order['action']} {order.get('project')}: {'done' if result['ok'] else 'FAILED: ' + result['output']}") + + +def next_nightly(now): + today = now.replace(hour=HOUR, minute=0, second=0, microsecond=0) + return today if now < today else today + datetime.timedelta(days=1) + + +def serve(): + for folder in (REQUESTS, WORK): + folder.mkdir(parents=True, exist_ok=True) + for interrupted in REQUESTS.glob("*.running"): + interrupted.with_suffix(".result").write_text(json.dumps({**json.loads(interrupted.read_text(encoding="utf-8")), "ok": False, "output": "interrupted by a restart"}), encoding="utf-8") + interrupted.unlink() + due = next_nightly(datetime.datetime.now(datetime.UTC)) + log(f"waiting for requests; the nightly run is at {due:%Y-%m-%d %H:%M}Z") + while True: + for request in sorted(REQUESTS.glob("*.request")): + handle(request) + if datetime.datetime.now(datetime.UTC) >= due: + run() + due = next_nightly(datetime.datetime.now(datetime.UTC)) + time.sleep(5) + + +if __name__ == "__main__": + sys.exit(serve()) diff --git a/edge/backup.yaml b/edge/backup.yaml new file mode 100644 index 0000000..6e5f42f --- /dev/null +++ b/edge/backup.yaml @@ -0,0 +1,23 @@ +services: + backup: + image: python:3.14-alpine + command: [python, /app/backup.py] + environment: + KEEP_DAYS: "14" + BACKUP_HOUR: "3" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /var/lib/docker/volumes:/volumes + - /var/lib/graphite/backups:/backups + - ./files/backup.py:/app/backup.py:ro + restart: unless-stopped + logging: + driver: json-file + options: {max-size: 10m, max-file: "5"} + security_opt: + - no-new-privileges:true + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M diff --git a/edge/wake.yaml b/edge/wake.yaml deleted file mode 100644 index fd5769e..0000000 --- a/edge/wake.yaml +++ /dev/null @@ -1,18 +0,0 @@ -name: wake - -services: - ofelia: - image: mcuadros/ofelia:0.3.22 - command: [daemon, --config=/etc/ofelia.ini] - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - - ./ofelia.ini:/etc/ofelia.ini:ro - restart: unless-stopped - logging: - driver: json-file - options: {max-size: 10m, max-file: "5"} - deploy: - resources: - limits: - cpus: "0.1" - memory: 32M diff --git a/host/agent.py b/host/agent.py index 6d6c742..7ba86b5 100644 --- a/host/agent.py +++ b/host/agent.py @@ -13,7 +13,7 @@ CONSOLE_TOKEN = os.environ.get("CONSOLE_TOKEN", "").encode() ENV = pathlib.Path("/etc/graphite") NAME = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,38}[a-z0-9])?$") -FILES = {"compose.yaml", "traefik.yml", "ofelia.ini", "sleep"} +FILES = re.compile(r"^(compose\.yaml|traefik\.yml|values|files/[A-Za-z0-9._-]+)$") SKEW = 300 @@ -51,22 +51,25 @@ def keys(path): return {line.split("=", 1)[0] for line in path.read_text(encoding="utf-8").splitlines() if "=" in line} +def compose_options(name): + values = BASE / name / "values" + return [*(["--env-file", str(values)] if values.exists() else []), "--env-file", str(env_file(name))] + + def up(name, request): folder = BASE / name - unknown = sorted(set(request.get("files", {})) - FILES) + unknown = sorted(file for file in request.get("files", {}) if not FILES.match(file)) if unknown: raise ValueError(f"{', '.join(unknown)} is not a file the agent writes") + missing = sorted(set(request.get("require", [])) - keys(env_file(name))) + if missing: + raise LookupError(f"{name} is waiting on secrets: {', '.join(missing)}") for file, content in request.get("files", {}).items(): if content is None: (folder / file).unlink(missing_ok=True) else: write(folder / file, content) - missing = sorted(set(request.get("require", [])) - keys(env_file(name))) - if missing: - raise LookupError(f"{name} is waiting on secrets: {', '.join(missing)}") output = run(str(BASE / "bin" / "graphite-up"), name) - if request.get("restart"): - output += run("docker", "compose", "--project-directory", str(folder), "-p", name, "restart") if "route" in request: route = BASE / "edge" / "routes" / f"{name}.yml" if request["route"] is None: @@ -79,13 +82,9 @@ def up(name, request): def down(name): folder = BASE / name (BASE / "edge" / "routes" / f"{name}.yml").unlink(missing_ok=True) - (folder / "sleep").unlink(missing_ok=True) if not (folder / "compose.yaml").exists(): return f"{name} is not deployed." - return run( - "docker", "compose", "--project-directory", str(folder), "-p", name, - "--env-file", str(env_file(name)), "down", "--remove-orphans", - ) + return run("docker", "compose", "--project-directory", str(folder), "-p", name, *compose_options(name), "down", "--remove-orphans") def reapply(name): diff --git a/host/graphite-up b/host/graphite-up index 7200016..d8a7311 100755 --- a/host/graphite-up +++ b/host/graphite-up @@ -9,10 +9,9 @@ if [ ! -f "$env" ]; then touch secrets.env chmod 600 secrets.env fi -docker compose -p "$name" --env-file "$env" up -d --wait --wait-timeout 300 --remove-orphans -docker image prune -af --filter until=24h >/dev/null || echo "graphite-up: image prune failed" >&2 - -if [ -f sleep ]; then - curl -fsS -o /dev/null --max-time 90 \ - "http://127.0.0.1:10000/api/strategies/blocking?group=$name&session_duration=$(cat sleep)&timeout=1m" +set -- +if [ -f values ]; then + set -- --env-file values fi +docker compose -p "$name" "$@" --env-file "$env" up -d --wait --wait-timeout 300 --remove-orphans +docker image prune -af --filter until=24h >/dev/null || echo "graphite-up: image prune failed" >&2 diff --git a/people/amrmscll.yml b/people/amrmscll.yml deleted file mode 100644 index 148f0f1..0000000 --- a/people/amrmscll.yml +++ /dev/null @@ -1,32 +0,0 @@ -handle: amrmscll -github: AmrMsCLL -profile: manual -name: Amr El-Sheraey -headline: Full-stack Software Engineer -bio: > - Computer Science and AI graduate of FCAI, Cairo University, building document-intelligence - software at Syntheia. A normal week touches a TypeScript API, a React front end, a Python ML service and a Helm chart. Most useful where those meet, and most interested in whichever part I have not done before. -stack: - - group: languages - items: [TypeScript, JavaScript, Python, C++, SQL] - - group: backend - items: [NestJS, Express, FastAPI, Django, Prisma, GraphQL] - - group: frontend - items: [React, Redux Toolkit, Tailwind, MUI, Vite, Office.js] - - group: data - items: [PostgreSQL, MongoDB, Redis, Qdrant, Milvus] - - group: ml - items: [PyTorch, Transformers, spaCy, scikit-learn, Celery] - - group: platform - items: [Docker, Kubernetes, Helm, Terraform, AWS, Azure] - - group: testing - items: [Jest, Pytest, Playwright, SonarQube] -links: - - label: github - url: https://github.com/AmrMsCLL - - label: linkedin - url: https://www.linkedin.com/in/amrsherieycll - - label: email - url: mailto:amrsherieycll@gmail.com - - label: discord - url: https://discord.com/users/412317378337046529 diff --git a/people/bluocaroot.yml b/people/bluocaroot.yml deleted file mode 100644 index a3f0d6c..0000000 --- a/people/bluocaroot.yml +++ /dev/null @@ -1,30 +0,0 @@ -handle: bluocaroot -github: BluoCaroot -profile: manual -name: Assem Amr Abdelfattah -headline: Software Engineer at Syntheia -bio: > - Software engineer from Egypt, building document-intelligence software at Syntheia: a NestJS - API, a React portal and an Outlook add-in on Office.js. Came up through competitive programming, - which still shapes how I approach problems, and outside work I build tools like Digital to - Physical to learn the parts of the stack I have not touched yet. -stack: - - group: languages - items: [TypeScript, JavaScript, C++, Python, C, SQL] - - group: backend - items: [NestJS, Node.js, Express, GraphQL] - - group: frontend - items: [React, MUI, Tailwind, Vite, Office.js] - - group: data - items: [PostgreSQL, MongoDB] - - group: platform - items: [Docker, Azure, Linux, Git] - - group: testing - items: [Jest, SonarQube] -links: - - label: github - url: https://github.com/BluoCaroot - - label: linkedin - url: https://www.linkedin.com/in/assem-amr-abdelfattah/ - - label: codeforces - url: https://codeforces.com/profile/BluoCaroot diff --git a/people/mahmoudhanyfathalla.yml b/people/mahmoudhanyfathalla.yml deleted file mode 100644 index fde1d14..0000000 --- a/people/mahmoudhanyfathalla.yml +++ /dev/null @@ -1,7 +0,0 @@ -handle: mahmoudhanyfathalla -github: MahmoudHanyFathalla -profile: github -name: Mahmoud Hany Fathalla -links: -- label: github - url: https://github.com/MahmoudHanyFathalla diff --git a/people/nier0o.yml b/people/nier0o.yml deleted file mode 100644 index d9ae460..0000000 --- a/people/nier0o.yml +++ /dev/null @@ -1,7 +0,0 @@ -handle: nier0o -github: Nier0o -profile: github -name: Ahmed Saber -links: -- label: github - url: https://github.com/Nier0o diff --git a/people/omar-mohamed24.yml b/people/omar-mohamed24.yml deleted file mode 100644 index 0c0ebb6..0000000 --- a/people/omar-mohamed24.yml +++ /dev/null @@ -1,34 +0,0 @@ -handle: omar-mohamed24 -github: Omar-Mohamed24 -profile: manual -name: Omar Mohamed -headline: Backend & Full-stack Software Engineer -bio: > - Software engineer who goes looking for the hard parts. Wrote a Game Boy emulator from scratch in - C++20, compiled it to WebAssembly and shipped it as PixelPocket, where it passes all of Blargg's - CPU tests and plays in any browser. Built GameForge, a ten-service platform where an LLM planner, - an MCP asset pipeline and a Godot build step turn one sentence into a playable game. Codeforces - Expert with 1000+ problems solved, CS graduate of FCAI, Cairo University, and formerly a prompt - engineer at Scale AI. -stack: - - group: languages - items: [C++, JavaScript, Python, Java, C#] - - group: backend - items: [Node.js, Express, Django, REST APIs, Microservices] - - group: frontend - items: [React, WebAssembly, HTML, CSS] - - group: data - items: [MongoDB, MySQL, MinIO] - - group: ai - items: [LLM Integration, MCP, Stability AI, Prompt Engineering] - - group: platform - items: [Docker, Docker Compose, GitHub Actions, Linux, Git] -links: - - label: github - url: https://github.com/Omar-Mohamed24 - - label: linkedin - url: https://www.linkedin.com/in/omar-mohamed-8034032a3 - - label: email - url: mailto:omarmohamed67371@gmail.com - - label: codeforces - url: https://codeforces.com/profile/Maliiketh diff --git a/schema/person.schema.json b/schema/person.schema.json deleted file mode 100644 index fa0c8a2..0000000 --- a/schema/person.schema.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/project-graphite/platform/blob/main/schema/person.schema.json", - "title": "Project Graphite person entry", - "type": "object", - "additionalProperties": false, - "required": ["handle", "github", "name"], - "properties": { - "handle": { - "type": "string", - "pattern": "^[a-z0-9](?:[a-z0-9-]{0,30}[a-z0-9])?$", - "description": "URL segment for this person. project-graphite.com/members/" - }, - "github": { - "type": "string", - "pattern": "^[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?$" - }, - "profile": { "enum": ["github", "manual"] }, - "name": { "type": "string", "minLength": 2, "maxLength": 60 }, - "headline": { "type": "string", "minLength": 4, "maxLength": 100 }, - "bio": { "type": "string", "minLength": 20, "maxLength": 600 }, - "stack": { - "type": "array", - "maxItems": 8, - "items": { - "type": "object", - "additionalProperties": false, - "required": ["group", "items"], - "properties": { - "group": { "type": "string", "minLength": 2, "maxLength": 20 }, - "items": { - "type": "array", - "minItems": 1, - "maxItems": 12, - "uniqueItems": true, - "items": { "type": "string", "minLength": 1, "maxLength": 24 } - } - } - } - }, - "links": { - "type": "array", - "maxItems": 6, - "uniqueItems": true, - "items": { - "type": "object", - "additionalProperties": false, - "required": ["label", "url"], - "properties": { - "label": { "type": "string", "minLength": 2, "maxLength": 20 }, - "url": { "type": "string", "pattern": "^(https://|mailto:)" } - } - } - } - } -} diff --git a/scripts/check-memory.py b/scripts/check-memory.py new file mode 100644 index 0000000..b652663 --- /dev/null +++ b/scripts/check-memory.py @@ -0,0 +1,48 @@ +import re +import sys + +import yaml + +import deploy + +UNITS = {"": 1, "b": 1, "k": 1024, "m": 1024**2, "g": 1024**3} + + +def mebibytes(value): + amount, unit = re.fullmatch(r"([0-9]+)([bkmg]?)b?", str(value).lower()).groups() + return int(amount) * UNITS[unit] // 1024**2 + + +def main(): + rows = [] + problems = [] + for slug in filter(deploy.compose_project, deploy.projects()): + try: + config = deploy.render(slug)[0] + except ValueError as error: + problems.append(str(error)) + continue + resources = {name: service.get("deploy", {}).get("resources", {}) for name, service in config["services"].items()} + problems.extend(f"deploy/{slug}: {name} has no memory limit" for name, entry in resources.items() if not entry.get("limits", {}).get("memory")) + rows.append(( + slug, + sum(mebibytes(entry.get("limits", {}).get("memory", 0)) for entry in resources.values()), + sum(mebibytes(entry.get("reservations", {}).get("memory", 0)) for entry in resources.values()), + )) + + host = yaml.safe_load(deploy.read(deploy.DEPLOY / "host.yaml")) + budget = mebibytes(host["memory"]) - mebibytes(host["system"]) + reserved = sum(row[2] for row in rows) + print(f"{'project':<24}{'limit':>8}{'reserve':>9}") + for slug, limit, reserve in rows: + print(f"{slug:<24}{limit:>7}M{reserve:>8}M") + print(f"{'total':<24}{sum(row[1] for row in rows):>7}M{reserved:>8}M budget {budget}M") + if reserved > budget: + problems.append(f"reserved memory {reserved}M exceeds the {budget}M budget in deploy/host.yaml") + for problem in problems: + print(f"error: {problem}", file=sys.stderr) + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check-values.py b/scripts/check-values.py new file mode 100644 index 0000000..73dbd34 --- /dev/null +++ b/scripts/check-values.py @@ -0,0 +1,58 @@ +import re +import sys + +import deploy + +REFERENCE = re.compile(r"(?", file=sys.stderr) - return 1 - - projects = [project_card(p) for p in load("projects") if p.get("listed", True)] - projects.sort(key=lambda entry: entry["name"].lower()) - - people = [person_card(p) for p in load("people")] - people.sort(key=lambda entry: entry["name"].lower()) - - pathlib.Path(sys.argv[1]).write_text( - json.dumps({"members": people, "projects": projects}, indent=2) + "\n", encoding="utf-8" - ) - print(f"Compiled {len(people)} members and {len(projects)} projects to {sys.argv[1]}.") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/deploy.py b/scripts/deploy.py index 2c4f6cb..63681fa 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -1,15 +1,10 @@ -import io import json import os import pathlib import re -import shlex -import shutil import subprocess import sys -import tarfile import tempfile -import time import urllib.request import yaml @@ -19,93 +14,93 @@ ROOT = pathlib.Path(__file__).resolve().parent.parent DEPLOY = ROOT / "deploy" -CHART = ROOT / "charts" / "core" -CURL = "curlimages/curl:8.22.0" +EDGE = ROOT / "edge" DEPLOY_HOST = "deploy.project-graphite.com" -COMPONENTS = ("postgres", "redis", "backup") -CHANGE = re.compile(r"^(?:deploy|tags|projects)/([a-z0-9-]+)(?:/|[.]json$|[.]yml$)") -REFERENCE = re.compile(r"(?8}{'reserve':>9} enabled") - for slug, limit, reserve, on in rows: - print(f"{slug:<24}{limit:>7}M{reserve:>8}M {'yes' if on else 'no'}") - print(f"{'total':<24}{sum(row[1] for row in rows):>7}M{reserved:>8}M budget {budget}M") - if reserved > budget: - problems.append(f"reserved memory {reserved}M exceeds the {budget}M budget in deploy/host.yaml") - + problems.extend(f"deploy/{slug}/values: {name} is not used by a Pages site" for name in set(values(slug)) - {"IMAGE_TAG"}) + elif kind == "compose" and compose_project(slug): + route(slug, render(slug)[0]) + else: + problems.append(f"deploy/{slug}: needs compose.yaml and deploy.kind compose in projects/{slug}.yml, or deploy.kind pages") + except ValueError as error: + problems.append(str(error)) for problem in problems: print(f"error: {problem}", file=sys.stderr) + print(f"Checked {len(projects())} projects, found {len(problems)} problems.") return 1 if problems else 0 COMMANDS = { - "release-core": release_core, "plan": show_plan, "apply": apply, "check": check, diff --git a/scripts/people.py b/scripts/people.py deleted file mode 100644 index 600114c..0000000 --- a/scripts/people.py +++ /dev/null @@ -1,91 +0,0 @@ -import json -import sys -import urllib.request - -import yaml - -import sync -import validate - -LABELS = {"generic": "website"} - - -def get(path): - with urllib.request.urlopen(sync.request(path)) as response: - return json.load(response) - - -def members(): - page = 1 - while True: - entries = get(f"/orgs/{sync.ORGANIZATION}/members?per_page=100&page={page}") - yield from entries - if len(entries) < 100: - return - page += 1 - - -def links(user): - found = [{"label": "github", "url": user["html_url"]}] - if user.get("blog"): - blog = user["blog"] if "://" in user["blog"] else f"https://{user['blog']}" - found.append({"label": "website", "url": blog}) - found.extend( - {"label": LABELS.get(account["provider"], account["provider"]), "url": account["url"]} - for account in get(f"/users/{user['login']}/social_accounts") - ) - - unique = [] - for link in found: - if link["url"].startswith("https://") and link not in unique: - unique.append(link) - return unique[:6] - - -def from_github(person, user): - profile = {key: value for key, value in person.items() if key != "headline"} - profile["name"] = user["name"] or user["login"] - headline = " ".join((user["bio"] or "").split()) - if 4 <= len(headline) <= 100: - profile["headline"] = headline - profile["links"] = links(user) - return profile - - -def main(): - people = { - person["github"].lower(): (path, person) - for path, person in validate.load("people") - if isinstance(person, dict) and isinstance(person.get("github"), str) - } - changed = [] - - for member in members(): - login = member["login"] - path, person = people.get( - login.lower(), - ( - sync.ROOT / "people" / f"{login.lower()}.yml", - {"handle": login.lower(), "github": login, "profile": "github"}, - ), - ) - if person.get("profile") == "manual": - continue - - updated = from_github(person, get(f"/users/{login}")) - if updated != person: - path.write_text( - yaml.safe_dump(updated, sort_keys=False, allow_unicode=True, width=100), - encoding="utf-8", - ) - changed.append(updated["handle"]) - - if changed: - print(f"Updated {len(changed)} people from GitHub profiles: {', '.join(sorted(changed))}.") - else: - print("People already match their GitHub profiles.") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/sync.py b/scripts/sync.py index e0bc59f..e36fd20 100644 --- a/scripts/sync.py +++ b/scripts/sync.py @@ -1,5 +1,4 @@ import json -import os import pathlib import re import sys @@ -11,43 +10,18 @@ import validate ROOT = pathlib.Path(__file__).resolve().parent.parent -ORGANIZATION = "Project-Graphite" SLUG = json.loads((ROOT / "schema" / "project.schema.json").read_text(encoding="utf-8"))["properties"]["slug"]["pattern"] -def request(path, accept="application/vnd.github+json"): - token = os.environ.get("GH_TOKEN") - if not token: - raise RuntimeError("GH_TOKEN is required") - return urllib.request.Request( - f"https://api.github.com{path}", - headers={ - "Accept": accept, - "Authorization": f"Bearer {token}", - "X-GitHub-Api-Version": "2022-11-28", - "User-Agent": "project-graphite-registry-sync", - }, - ) - - def repositories(): - page = 1 - while True: - with urllib.request.urlopen( - request(f"/orgs/{ORGANIZATION}/repos?type=all&per_page=100&page={page}") - ) as response: - entries = json.load(response) - yield from entries - if len(entries) < 100: - return - page += 1 + return validate.paginate(f"/orgs/{validate.ORGANIZATION}/repos?type=all") def manifest(repository): try: with urllib.request.urlopen( - request( - f"/repos/{ORGANIZATION}/{repository}/contents/.graphite.yml", + validate.request( + f"/repos/{validate.ORGANIZATION}/{repository}/contents/.graphite.yml", "application/vnd.github.raw+json", ) ) as response: @@ -66,7 +40,7 @@ def entries(projects): return [(ROOT / "projects" / f"{slug}.yml", project) for slug, project in sorted(projects.items())] -def apply(name, text, projects, organisation, people): +def apply(name, text, projects, organisation, members): try: source = yaml.safe_load(text) except yaml.YAMLError as error: @@ -103,9 +77,9 @@ def apply(name, text, projects, organisation, people): for slug, project in projects.items() if not (isinstance(project, dict) and project.get("repo") == repo) } | candidates - existing = set(validate.registry_problems(entries(projects), people)) + existing = set(validate.registry_problems(entries(projects), members)) problems.extend( - line for line in validate.registry_problems(entries(updated), people) if line not in existing + line for line in validate.registry_problems(entries(updated), members) if line not in existing ) return updated, problems @@ -115,25 +89,10 @@ def report(name, problems): print(f"error: {name}: {line}", file=sys.stderr) -def check(name, path): - _, problems = apply( - name, - pathlib.Path(path).read_text(encoding="utf-8"), - registry(), - {repository["name"].lower() for repository in repositories()}, - validate.load("people"), - ) - report(name, problems) - if problems: - return 1 - print(f"{path} is a valid manifest for {name}.") - return 0 - - def sync(): existing = registry() projects = existing - people = validate.load("people") + members = validate.members() listed = list(repositories()) organisation = {repository["name"].lower() for repository in listed} skipped = [] @@ -142,7 +101,7 @@ def sync(): text = manifest(repository["name"]) if text is None: continue - updated, problems = apply(repository["name"], text, projects, organisation, people) + updated, problems = apply(repository["name"], text, projects, organisation, members) if problems: report(repository["name"], problems) skipped.append(repository["name"]) @@ -174,14 +133,5 @@ def sync(): return 0 -def main(): - if len(sys.argv) == 1: - return sync() - if len(sys.argv) == 3: - return check(*sys.argv[1:]) - print("usage: sync.py [ ]", file=sys.stderr) - return 1 - - if __name__ == "__main__": - sys.exit(main()) + sys.exit(sync()) diff --git a/scripts/validate.py b/scripts/validate.py index e42c5ff..11ceaba 100644 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -1,19 +1,45 @@ import json +import os import pathlib import sys +import urllib.request import jsonschema import yaml ROOT = pathlib.Path(__file__).resolve().parent.parent +ORGANIZATION = "Project-Graphite" +PROJECT = jsonschema.Draft202012Validator(json.loads((ROOT / "schema" / "project.schema.json").read_text(encoding="utf-8"))) + + +def request(path, accept="application/vnd.github+json"): + token = os.environ.get("GH_TOKEN") + if not token: + raise RuntimeError("GH_TOKEN is required") + return urllib.request.Request( + f"https://api.github.com{path}", + headers={ + "Accept": accept, + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "project-graphite-registry-sync", + }, + ) -def validator_for(name): - return jsonschema.Draft202012Validator(json.loads((ROOT / "schema" / name).read_text(encoding="utf-8"))) +def paginate(path): + page = 1 + while True: + with urllib.request.urlopen(request(f"{path}{'&' if '?' in path else '?'}per_page=100&page={page}")) as response: + entries = json.load(response) + yield from entries + if len(entries) < 100: + return + page += 1 -PROJECT = validator_for("project.schema.json") -PERSON = validator_for("person.schema.json") +def members(): + return {member["login"].lower() for member in paginate(f"/orgs/{ORGANIZATION}/members")} def load(directory): @@ -63,16 +89,11 @@ def uniqueness_problems(entries, value_of, label): claimed[key] = path.name -def ownership_problems(projects, people): - handles = { - person["handle"] - for _, person in people - if isinstance(person, dict) and isinstance(person.get("handle"), str) - } +def ownership_problems(projects, organisation): for path, project in projects: owner = project.get("owner") if isinstance(project, dict) else None - if owner and isinstance(owner, str) and owner not in handles: - yield f"{path.name}: owner {owner} has no entry in people/" + if owner and isinstance(owner, str) and owner.lower() not in organisation: + yield f"{path.name}: owner {owner} is not a member of {ORGANIZATION}" def hierarchy_problems(projects): @@ -102,17 +123,9 @@ def hierarchy_problems(projects): yield f"{path.name}: child project must use parent repository {parent.get('repo')}" -def registry_problems(projects, people): - problems = [line for path, person in people for line in schema_problems(PERSON, path, person, "handle")] - problems.extend(line for path, project in projects for line in project_problems(path, project)) - problems.extend( - uniqueness_problems( - people, - lambda person: person.get("github") if isinstance(person, dict) else None, - "github account", - ) - ) - problems.extend(ownership_problems(projects, people)) +def registry_problems(projects, organisation): + problems = [line for path, project in projects for line in project_problems(path, project)] + problems.extend(ownership_problems(projects, organisation)) problems.extend(hierarchy_problems(projects)) problems.extend( uniqueness_problems( @@ -126,20 +139,15 @@ def registry_problems(projects, people): def main(): projects = load("projects") - people = load("people") - if not projects: print("No registry entries in projects/.", file=sys.stderr) return 1 - if not people: - print("No entries in people/.", file=sys.stderr) - return 1 - problems = registry_problems(projects, people) + problems = registry_problems(projects, members()) for line in problems: print(f"error: {line}", file=sys.stderr) - print(f"Checked {len(people)} people and {len(projects)} projects, found {len(problems)} problems.") + print(f"Checked {len(projects)} projects, found {len(problems)} problems.") return 1 if problems else 0 diff --git a/shared/app.yaml b/shared/app.yaml new file mode 100644 index 0000000..d5bf087 --- /dev/null +++ b/shared/app.yaml @@ -0,0 +1,32 @@ +services: + service: + init: true + restart: unless-stopped + logging: + driver: json-file + options: {max-size: 10m, max-file: "5"} + security_opt: + - no-new-privileges:true + healthcheck: + interval: 30s + timeout: 5s + retries: 3 + start_period: 0s + deploy: + resources: + limits: {cpus: "0.5", memory: 256M} + reservations: {memory: 64M} + + app: + extends: service + expose: ["${APP_PORT:-3000}"] + environment: + APP_URL: https://${HOST} + networks: + default: {} + edge: + aliases: ["${PROJECT}"] + deploy: + resources: + limits: {cpus: "${APP_CPUS:-0.5}", memory: "${APP_MEMORY:-256M}"} + reservations: {memory: "${APP_RESERVE:-64M}"} diff --git a/shared/postgres.yaml b/shared/postgres.yaml new file mode 100644 index 0000000..06af269 --- /dev/null +++ b/shared/postgres.yaml @@ -0,0 +1,20 @@ +services: + postgres: + extends: {file: app.yaml, service: service} + image: postgres:18-alpine + command: [postgres, -c, "shared_buffers=${POSTGRES_SHARED_BUFFERS:-128MB}", -c, "max_connections=${POSTGRES_MAX_CONNECTIONS:-100}"] + environment: + POSTGRES_DB: ${PROJECT} + POSTGRES_USER: ${PROJECT} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?} + volumes: [postgres-data:/var/lib/postgresql] + labels: {graphite.backup.postgres: "true"} + healthcheck: + test: [CMD-SHELL, "pg_isready -U \"$$POSTGRES_USER\" -d \"$$POSTGRES_DB\""] + interval: 10s + retries: 10 + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "${POSTGRES_CPUS:-0.5}", memory: "${POSTGRES_MEMORY:-256M}"} + reservations: {memory: "${POSTGRES_RESERVE:-64M}"} diff --git a/shared/redis-password.yaml b/shared/redis-password.yaml new file mode 100644 index 0000000..bc95abf --- /dev/null +++ b/shared/redis-password.yaml @@ -0,0 +1,8 @@ +services: + redis: + extends: {file: redis.yaml, service: redis} + command: [sh, -c, "exec redis-server --appendonly yes --maxmemory ${REDIS_MAXMEMORY:-48mb} --maxmemory-policy ${REDIS_POLICY:-allkeys-lru} --requirepass \"$$REDIS_PASSWORD\""] + environment: + REDIS_PASSWORD: ${REDIS_PASSWORD:?} + healthcheck: + test: [CMD-SHELL, "REDISCLI_AUTH=\"$$REDIS_PASSWORD\" redis-cli ping | grep -q PONG"] diff --git a/shared/redis.yaml b/shared/redis.yaml new file mode 100644 index 0000000..3757a09 --- /dev/null +++ b/shared/redis.yaml @@ -0,0 +1,14 @@ +services: + redis: + extends: {file: app.yaml, service: service} + image: redis:8.10-alpine + command: [redis-server, --appendonly, "yes", --maxmemory, "${REDIS_MAXMEMORY:-48mb}", --maxmemory-policy, "${REDIS_POLICY:-allkeys-lru}"] + volumes: [redis-data:/data] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 10s + retries: 10 + deploy: + resources: + limits: {cpus: "0.25", memory: "${REDIS_MEMORY:-64M}"} + reservations: {memory: "${REDIS_RESERVE:-16M}"} diff --git a/tags/console.json b/tags/console.json deleted file mode 100644 index 165b401..0000000 --- a/tags/console.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "app": "sha-8d9bb17" -} diff --git a/tags/digital-to-physical.json b/tags/digital-to-physical.json deleted file mode 100644 index cd6d273..0000000 --- a/tags/digital-to-physical.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "site": "sha-d5ba7b2" -} diff --git a/tags/graphite-hub.json b/tags/graphite-hub.json deleted file mode 100644 index 185df70..0000000 --- a/tags/graphite-hub.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "site": "sha-ff5f3e8" -} diff --git a/tags/graphite-message-lab.json b/tags/graphite-message-lab.json deleted file mode 100644 index 06f012e..0000000 --- a/tags/graphite-message-lab.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "web": "sha-599fbe4", - "worker": "sha-599fbe4" -} diff --git a/tags/graphite-tracker.json b/tags/graphite-tracker.json deleted file mode 100644 index 483ab49..0000000 --- a/tags/graphite-tracker.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "app": "sha-7533f37" -} diff --git a/tags/pixelpocket.json b/tags/pixelpocket.json deleted file mode 100644 index 67a977c..0000000 --- a/tags/pixelpocket.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "backend": "sha-e38c8e1", - "frontend": "sha-e38c8e1" -} diff --git a/tags/strata.json b/tags/strata.json deleted file mode 100644 index 5f25170..0000000 --- a/tags/strata.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "app": "sha-ebcde55" -}