diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index b7d400a..65a2bdc 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -4,12 +4,11 @@ on: push: branches: [main] paths: - - charts/core/** - deploy/** - edge/** - host/** - projects/** - - tags/** + - shared/** workflow_dispatch: inputs: project: @@ -36,8 +35,6 @@ jobs: plan: name: Plan runs-on: ubuntu-latest - permissions: - contents: write outputs: pages: ${{ steps.plan.outputs.pages }} steps: @@ -48,7 +45,6 @@ jobs: with: python-version: "3.14" - run: pip install pyyaml - - run: python scripts/deploy.py release-core - id: plan run: python scripts/deploy.py plan | tee -a "$GITHUB_OUTPUT" @@ -66,9 +62,6 @@ jobs: with: python-version: "3.14" - run: pip install pyyaml - - uses: azure/setup-helm@v5 - with: - version: v3.22.0 - run: python scripts/deploy.py apply env: DEPLOY_TOKEN: ${{ secrets.DEPLOY_TOKEN }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index afe1ce4..0000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Main - -on: - push: - branches: [main] - paths: - - projects/** - - people/** - - schema/** - - scripts/** - -concurrency: - group: ${{ github.workflow }} - cancel-in-progress: false - -jobs: - publish: - name: Publish registry - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - - uses: actions/setup-python@v7 - with: - python-version: "3.14" - - - run: pip install pyyaml jsonschema - - - run: python scripts/validate.py - - - run: python scripts/compile.py registry.json - - - uses: actions/create-github-app-token@v3 - id: app-token - with: - client-id: ${{ secrets.PLATFORM_APP_CLIENT_ID }} - private-key: ${{ secrets.PLATFORM_APP_PRIVATE_KEY }} - owner: ${{ github.repository_owner }} - repositories: graphite-hub - - - name: Sync to Graphite Hub - env: - TOKEN: ${{ steps.app-token.outputs.token }} - OWNER: ${{ github.repository_owner }} - run: | - set -euo pipefail - - git clone --quiet --depth 1 "https://x-access-token:${TOKEN}@github.com/${OWNER}/graphite-hub.git" /tmp/graphite-hub - install -D registry.json /tmp/graphite-hub/site/src/data/registry.json - - cd /tmp/graphite-hub - git config user.name "graphite-registry" - git config user.email "registry@project-graphite.com" - git add site/src/data/registry.json - - if git diff --cached --quiet; then - echo "Registry unchanged; Graphite Hub already current." - exit 0 - fi - - git commit --message "chore: sync registry from platform@${GITHUB_SHA::7}" - git push - echo "Graphite Hub updated." diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index d0687fd..cf985a4 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -16,25 +16,28 @@ jobs: with: python-version: "3.14" - run: pip install pyyaml jsonschema + - uses: actions/create-github-app-token@v3 + id: app-token + with: + client-id: ${{ secrets.PLATFORM_APP_CLIENT_ID }} + private-key: ${{ secrets.PLATFORM_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} - run: python scripts/validate.py + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} deploy: name: Deploy values runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - with: - fetch-depth: 0 - uses: actions/setup-python@v7 with: python-version: "3.14" - run: pip install pyyaml - - uses: azure/setup-helm@v5 - with: - version: v3.22.0 - run: python scripts/deploy.py check - env: - BASE_REF: ${{ github.base_ref }} + - run: python scripts/check-memory.py + - run: python scripts/check-values.py secrets: name: Secrets diff --git a/.github/workflows/sync-registry.yml b/.github/workflows/sync-registry.yml index e6bdb98..06afbee 100644 --- a/.github/workflows/sync-registry.yml +++ b/.github/workflows/sync-registry.yml @@ -4,6 +4,8 @@ on: workflow_dispatch: repository_dispatch: types: [sync-registry] + schedule: + - cron: "17 * * * *" concurrency: group: sync-registry @@ -44,14 +46,9 @@ jobs: GH_TOKEN: ${{ steps.app-token.outputs.token }} run: python scripts/sync.py - - name: Sync people from GitHub profiles - id: people - continue-on-error: true + - run: python scripts/validate.py env: GH_TOKEN: ${{ steps.app-token.outputs.token }} - run: python scripts/people.py - - - run: python scripts/validate.py - name: Open registry update env: @@ -60,7 +57,7 @@ jobs: WORKFLOW_TOKEN: ${{ github.token }} run: | set -euo pipefail - git add projects people + git add projects if git diff --cached --quiet; then echo "Registry already current." exit 0 @@ -99,9 +96,9 @@ jobs: fi file_count="$(gh pr view "$pr" --json files --jq '.files | length')" - invalid_paths="$(gh pr view "$pr" --json files --jq '[.files[].path | select(test("^(projects|people)/[^/]+\\.yml$") | not)] | join(", ")')" + invalid_paths="$(gh pr view "$pr" --json files --jq '[.files[].path | select(test("^projects/[^/]+\\.yml$") | not)] | join(", ")')" if [ "$file_count" -eq 0 ] || [ -n "$invalid_paths" ]; then - echo "::error::Registry PR #${pr} changes files outside projects/*.yml and people/*.yml: ${invalid_paths}" + echo "::error::Registry PR #${pr} changes files outside projects/*.yml: ${invalid_paths}" exit 1 fi @@ -129,7 +126,7 @@ jobs: gh pr merge "$pr" --squash --delete-branch --match-head-commit "$head_sha" - name: Report skipped manifests - if: steps.sync.outcome == 'failure' || steps.people.outcome == 'failure' + if: steps.sync.outcome == 'failure' run: | - echo "::error::The sync skipped invalid repository manifests or could not read GitHub profiles; see the Sync repository manifests and Sync people steps." + echo "::error::The sync skipped invalid repository manifests; see the Sync repository manifests step." exit 1 diff --git a/.gitignore b/.gitignore index 157ae76..c18dd8d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1 @@ __pycache__/ -registry.json diff --git a/charts/core/Chart.yaml b/charts/core/Chart.yaml deleted file mode 100644 index d0260a7..0000000 --- a/charts/core/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v2 -name: core -description: Renders a Project Graphite project (deploy//core.yaml and values.yaml) into a Docker Compose project and its route. -type: application -version: 2.0.0 diff --git a/charts/core/files/backup.sh b/charts/core/files/backup.sh deleted file mode 100644 index 5beba23..0000000 --- a/charts/core/files/backup.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -set -eu -while true; do - stamp=$(date -u +%Y%m%dT%H%M%SZ) - dump="/backups/$PGDATABASE-$stamp.dump" - pg_dump -Fc -f "$dump.partial" - pg_restore --list "$dump.partial" > /dev/null - mv "$dump.partial" "$dump" - for volume in /data/*; do - if [ -d "$volume" ]; then - tar -czf "/backups/$(basename "$volume")-$stamp.tar.gz" -C /data "$(basename "$volume")" - fi - done - find /backups -type f -mtime "+$BACKUP_KEEP_DAYS" -delete - sleep 86400 -done diff --git a/charts/core/templates/_helpers.tpl b/charts/core/templates/_helpers.tpl deleted file mode 100644 index 96e0262..0000000 --- a/charts/core/templates/_helpers.tpl +++ /dev/null @@ -1,153 +0,0 @@ -{{- define "core.image" -}} -{{- if or (contains ":" .service.image) (contains "@" .service.image) -}} -{{ .service.image }} -{{- else -}} -{{ .service.image }}:{{ .service.tag | default (index .tags .name) | default "latest" }} -{{- end -}} -{{- end -}} - -{{- define "core.megabytes" -}} -{{- $amount := regexFind "^[0-9]+" . | atoi -}} -{{- if hasSuffix "G" . }}{{ mul $amount 1024 }}{{ else }}{{ $amount }}{{ end -}} -{{- end -}} - -{{- define "core.database" -}} -{{- .Values.postgres.database | default (replace "-" "_" .Values.slug) -}} -{{- end -}} - -{{- define "core.postgres" -}} -{{- $postgres := .Values.postgres -}} -{{- $database := include "core.database" . -}} -{{- $user := $postgres.user | default $database -}} -image: {{ $postgres.image }} -{{- with $postgres.settings }} -command: - - postgres - {{- range $key, $value := . }} - - -c - - {{ printf "%s=%v" $key $value | toRawJson }} - {{- end }} -{{- end }} -env: - {{- toYaml (merge (dict "POSTGRES_DB" $database "POSTGRES_USER" $user "POSTGRES_PASSWORD" "${POSTGRES_PASSWORD}") $postgres.env) | nindent 2 }} -volumes: [postgres-data:/var/lib/postgresql] -health: - test: [CMD-SHELL, {{ printf "pg_isready -U %s -d %s" $user $database | toRawJson }}] - interval: 10s - retries: 10 -stopGracePeriod: 30s -memory: {{ $postgres.memory }} -cpus: {{ $postgres.cpus | quote }} -{{- with $postgres.reserve }} -reserve: {{ . }} -{{- end }} -{{- end -}} - -{{- define "core.redis" -}} -{{- $redis := .Values.redis -}} -{{- $maxmemory := $redis.maxmemory | default (printf "%dmb" (div (mul (include "core.megabytes" $redis.memory | atoi) 3) 4)) -}} -{{- $flags := list "--appendonly" "yes" "--maxmemory" $maxmemory "--maxmemory-policy" (ternary "allkeys-lru" "noeviction" (ne (toString $redis.evict) "false")) -}} -image: {{ $redis.image }} -{{- if $redis.password }} -command: [sh, -c, {{ printf "exec redis-server %s --requirepass \"$$REDIS_PASSWORD\"" (join " " $flags) | toRawJson }}] -env: - REDIS_PASSWORD: ${REDIS_PASSWORD} -health: - test: [CMD-SHELL, "REDISCLI_AUTH=\"$$REDIS_PASSWORD\" redis-cli ping | grep -q PONG"] - interval: 10s - retries: 10 -{{- else }} -command: {{ toRawJson (prepend $flags "redis-server") }} -health: - test: [CMD, redis-cli, ping] - interval: 10s - retries: 10 -{{- end }} -volumes: [redis-data:/data] -memory: {{ $redis.memory }} -cpus: {{ $redis.cpus | quote }} -{{- with $redis.reserve }} -reserve: {{ . }} -{{- end }} -{{- end -}} - -{{- define "core.backup" -}} -{{- $backup := .Values.backup -}} -{{- $database := include "core.database" . -}} -image: {{ .Values.postgres.image }} -command: [sh, /scripts/backup.sh] -env: - PGHOST: postgres - PGUSER: {{ .Values.postgres.user | default $database }} - PGDATABASE: {{ $database }} - PGPASSWORD: ${POSTGRES_PASSWORD} - BACKUP_KEEP_DAYS: {{ $backup.keepDays | quote }} -dependsOn: {postgres: healthy} -volumes: - - backups:/backups - {{- range $backup.include }} - - {{ printf "%s:/data/%s:ro" . . }} - {{- end }} -files: - /scripts/backup.sh: {{ $.Files.Get "files/backup.sh" | toRawJson }} -memory: {{ $backup.memory }} -cpus: {{ $backup.cpus | quote }} -{{- with $backup.reserve }} -reserve: {{ . }} -{{- end }} -{{- end -}} - -{{- define "core.services" -}} -{{- $values := .Values -}} -{{- $components := dict -}} -{{- if $values.postgres.enabled }}{{ $_ := set $components "postgres" (include "core.postgres" . | fromYaml) }}{{ end -}} -{{- if $values.redis.enabled }}{{ $_ := set $components "redis" (include "core.redis" . | fromYaml) }}{{ end -}} -{{- if $values.backup.enabled }}{{ $_ := set $components "backup" (include "core.backup" . | fromYaml) }}{{ end -}} -{{- $shared := dict -}} -{{- $waits := dict -}} -{{- if $values.postgres.enabled -}} -{{- $database := include "core.database" . -}} -{{- $user := $values.postgres.user | default $database -}} -{{- $_ := set $shared "DATABASE_URL" (printf "postgresql://%s:${POSTGRES_PASSWORD}@postgres:5432/%s" $user $database) -}} -{{- $_ := set $shared "POSTGRES_HOST" "postgres" -}} -{{- $_ := set $shared "POSTGRES_PORT" "5432" -}} -{{- $_ := set $shared "POSTGRES_DB" $database -}} -{{- $_ := set $shared "POSTGRES_USER" $user -}} -{{- $_ := set $shared "POSTGRES_PASSWORD" "${POSTGRES_PASSWORD}" -}} -{{- $_ := set $waits "postgres" "healthy" -}} -{{- end -}} -{{- if $values.redis.enabled -}} -{{- $_ := set $shared "REDIS_URL" (ternary "redis://:${REDIS_PASSWORD}@redis:6379" "redis://redis:6379" (eq (toString $values.redis.password) "true")) -}} -{{- $_ := set $waits "redis" "healthy" -}} -{{- end -}} -{{- $all := dict -}} -{{- range $name, $service := $values.services -}} -{{- $env := deepCopy $shared -}} -{{- range $service.optional }}{{ $_ := set $env . (printf "${%s:-}" .) }}{{ end -}} -{{- range $service.secrets }}{{ $_ := set $env . (printf "${%s}" .) }}{{ end -}} -{{- range $key, $value := $service.env }}{{ $_ := set $env $key (toString $value) }}{{ end -}} -{{- $_ := set $all $name (merge (dict "env" $env "dependsOn" (merge (deepCopy ($service.dependsOn | default dict)) $waits)) (omit $service "env" "dependsOn" "secrets" "optional")) -}} -{{- end -}} -{{- range $name, $component := $components }}{{ $_ := set $all $name $component }}{{ end -}} -{{- toJson $all -}} -{{- end -}} - -{{- define "core.hosted" -}} -{{- $names := list -}} -{{- range $name, $service := (include "core.services" . | fromJson) -}} -{{- if $service.hosts }}{{ if not $service.port }}{{ fail (printf "%s sets hosts, so it must set port" $name) }}{{ end }}{{ $names = append $names $name }}{{ end -}} -{{- end -}} -{{- toJson (dict "names" $names) -}} -{{- end -}} - -{{- define "core.public" -}} -{{- $services := include "core.services" . | fromJson -}} -{{- $marked := list -}} -{{- $listening := list -}} -{{- range $name, $service := $services -}} -{{- if $service.public }}{{ $marked = append $marked $name }}{{ end -}} -{{- if $service.port }}{{ $listening = append $listening $name }}{{ end -}} -{{- end -}} -{{- if gt (len $marked) 1 }}{{ fail (printf "only one service can set public: true, not %s" (join ", " $marked)) }}{{ end -}} -{{- if $marked }}{{ first $marked }}{{ else if eq (len $listening) 1 }}{{ first $listening }}{{ end -}} -{{- end -}} diff --git a/charts/core/templates/compose.yaml b/charts/core/templates/compose.yaml deleted file mode 100644 index 555b960..0000000 --- a/charts/core/templates/compose.yaml +++ /dev/null @@ -1,134 +0,0 @@ -{{- $values := .Values -}} -{{- $conditions := dict "healthy" "service_healthy" "started" "service_started" "completed" "service_completed_successfully" -}} -{{- $volumes := dict -}} -{{- $configs := list -}} -{{- $services := include "core.services" . | fromJson -}} -{{- $public := include "core.public" . -}} -name: {{ $values.slug }} - -services: -{{- range $name, $service := $services }} - {{- $health := $service.health }} - {{- if kindIs "string" $health }}{{ $health = dict "path" $health }}{{ end }} - {{- $memory := $service.memory | default $values.service.memory }} - {{ $name }}: - image: {{ include "core.image" (dict "name" $name "service" $service "tags" $values.tags) }} - {{- with $service.entrypoint }} - entrypoint: {{ toRawJson . }} - {{- end }} - {{- with $service.command }} - command: {{ toRawJson . }} - {{- end }} - {{- with $service.env }} - environment: - {{- range $key, $value := . }} - {{ $key }}: {{ toString $value | toRawJson }} - {{- end }} - {{- end }} - {{- with $service.dependsOn }} - depends_on: - {{- range $dependency, $state := . }} - {{ $dependency }}: - condition: {{ get $conditions $state }} - {{- end }} - {{- end }} - {{- with $service.port }} - expose: - - {{ toString . | toRawJson }} - {{- end }} - {{- if or (eq $name $public) $service.hosts }} - networks: - default: {} - edge: - aliases: - {{- if eq $name $public }} - - {{ $values.slug }} - {{- end }} - {{- if $service.hosts }} - - {{ printf "%s-%s" $values.slug $name }} - {{- end }} - {{- end }} - {{- with $service.volumes }} - volumes: - {{- range . }} - - {{ toRawJson . }} - {{- $volume := first (splitList ":" .) }} - {{- if regexMatch "^[A-Za-z0-9][A-Za-z0-9_.-]*$" $volume }}{{ $_ := set $volumes $volume true }}{{ end }} - {{- end }} - {{- end }} - {{- with $service.files }} - configs: - {{- range $target, $content := . }} - {{- $config := printf "%s-%s" $name (base $target | replace "." "-") }} - {{- $configs = append $configs (dict "name" $config "content" $content) }} - - source: {{ $config }} - target: {{ $target }} - {{- end }} - {{- end }} - {{- with $health }} - healthcheck: - {{- if .path }} - {{- $url := printf "http://127.0.0.1:%v%s" $service.port .path }} - test: [CMD-SHELL, {{ printf "wget -q -O /dev/null %s || curl -fsS -o /dev/null %s" $url $url | toRawJson }}] - {{- else }} - test: {{ toRawJson .test }} - {{- end }} - interval: {{ .interval | default "30s" }} - timeout: {{ .timeout | default "5s" }} - retries: {{ .retries | default 3 }} - start_period: {{ .startPeriod | default "0s" }} - {{- end }} - init: {{ ne (toString $service.init) "false" }} - restart: {{ ternary "no" "unless-stopped" (default false $service.once) | quote }} - logging: - driver: json-file - options: {max-size: 10m, max-file: "5"} - {{- with $service.stopGracePeriod }} - stop_grace_period: {{ . }} - {{- end }} - {{- if $service.privileged }} - privileged: true - {{- end }} - {{- with $service.pid }} - pid: {{ . }} - {{- end }} - {{- if ne (toString $service.noNewPrivileges) "false" }} - security_opt: - - no-new-privileges:true - {{- end }} - {{- if and $values.sleep.after (not $service.once) }} - labels: - sablier.enable: "true" - sablier.group: {{ $values.slug }} - {{- end }} - deploy: - resources: - limits: - cpus: {{ toString ($service.cpus | default $values.service.cpus) | quote }} - memory: {{ $memory }} - {{- if not $service.once }} - reservations: - memory: {{ $service.reserve | default (printf "%dM" (div (include "core.megabytes" $memory | atoi) 4)) }} - {{- end }} -{{- end }} -{{- if $volumes }} - -volumes: - {{- range $volume, $_ := $volumes }} - {{ $volume }}: {} - {{- end }} -{{- end }} -{{- if $configs }} - -configs: - {{- range $configs }} - {{ .name }}: - content: {{ replace "$" "$$" .content | toRawJson }} - {{- end }} -{{- end }} -{{- if or $public (include "core.hosted" . | fromJson).names }} - -networks: - edge: - external: true -{{- end }} diff --git a/charts/core/templates/route.yaml b/charts/core/templates/route.yaml deleted file mode 100644 index 918471f..0000000 --- a/charts/core/templates/route.yaml +++ /dev/null @@ -1,59 +0,0 @@ -{{- $values := .Values -}} -{{- $public := include "core.public" . -}} -{{- if $values.host }} -{{- if not $public }}{{ fail "no service gets the hostname: give one service a port, or set public: true on one of them" }}{{ end }} -{{- $services := include "core.services" . | fromJson }} -{{- $service := index $services $public }} -{{- if not $service.port }}{{ fail (printf "%s gets the hostname, so it must set port" $public) }}{{ end }} -{{- $hosted := (include "core.hosted" . | fromJson).names }} -http: - routers: - {{ $values.slug }}: - rule: {{ printf "Host(`%s`)" $values.host | quote }} - entryPoints: [websecure] - service: {{ $values.slug }} - tls: - certResolver: letsencrypt - {{- if $values.sleep.after }} - middlewares: [{{ $values.slug }}-sleep] - {{- end }} - {{- range $name := $hosted }} - {{- range $index, $host := (index $services $name).hosts }} - {{ printf "%s-%s-%d" $values.slug $name $index }}: - rule: {{ printf "Host(`%s`)" $host | quote }} - entryPoints: [websecure] - service: {{ printf "%s-%s" $values.slug $name }} - tls: - certResolver: letsencrypt - {{- if $values.sleep.after }} - middlewares: [{{ $values.slug }}-sleep] - {{- end }} - {{- end }} - {{- end }} - services: - {{ $values.slug }}: - loadBalancer: - servers: - - url: {{ printf "http://%s:%v" $values.slug $service.port | quote }} - {{- range $name := $hosted }} - {{ printf "%s-%s" $values.slug $name }}: - loadBalancer: - servers: - - url: {{ printf "http://%s-%s:%v" $values.slug $name (index $services $name).port | quote }} - {{- end }} - {{- if $values.sleep.after }} - middlewares: - {{ $values.slug }}-sleep: - plugin: - sablier: - sablierUrl: http://sablier:10000 - group: {{ $values.slug }} - sessionDuration: {{ $values.sleep.after }} - ignoreUserAgent: - - "(?i)(bot|crawler|spider|preview|curl|wget|uptime)" - dynamic: - displayName: {{ $values.name | default $values.slug | quote }} - theme: graphite - refreshFrequency: 5s - {{- end }} -{{- end }} diff --git a/charts/core/values.schema.json b/charts/core/values.schema.json deleted file mode 100644 index 5b4954e..0000000 --- a/charts/core/values.schema.json +++ /dev/null @@ -1,152 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Project Graphite project", - "type": "object", - "additionalProperties": false, - "required": ["core", "slug", "services"], - "properties": { - "core": {"type": "string", "pattern": "^[0-9]+[.][0-9]+[.][0-9]+$"}, - "slug": {"type": "string", "pattern": "^[a-z0-9](?:[a-z0-9-]{0,38}[a-z0-9])?$"}, - "name": {"type": "string"}, - "host": {"type": "string"}, - "tags": {"type": "object", "additionalProperties": {"type": "string"}}, - "enabled": {"type": "boolean"}, - "proxied": {"type": "boolean"}, - "sleep": { - "type": "object", - "additionalProperties": false, - "properties": { - "after": {"$ref": "#/definitions/duration"}, - "wake": {"$ref": "#/definitions/cron"}, - "wakeFor": {"$ref": "#/definitions/duration"} - }, - "dependencies": {"wake": ["after"], "wakeFor": ["wake"]} - }, - "service": { - "type": "object", - "additionalProperties": false, - "properties": {"memory": {"$ref": "#/definitions/memory"}, "cpus": {"$ref": "#/definitions/cpus"}} - }, - "postgres": { - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": {"type": "boolean"}, - "image": {"type": "string", "minLength": 1}, - "database": {"$ref": "#/definitions/identifier"}, - "user": {"$ref": "#/definitions/identifier"}, - "settings": {"type": "object", "additionalProperties": {"type": ["string", "number"]}}, - "env": {"$ref": "#/definitions/env"}, - "memory": {"$ref": "#/definitions/memory"}, - "cpus": {"$ref": "#/definitions/cpus"}, - "reserve": {"$ref": "#/definitions/optionalMemory"} - } - }, - "redis": { - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": {"type": "boolean"}, - "image": {"type": "string", "minLength": 1}, - "password": {"type": "boolean"}, - "evict": {"type": "boolean"}, - "maxmemory": {"type": "string", "pattern": "^([0-9]+(mb|gb))?$"}, - "memory": {"$ref": "#/definitions/memory"}, - "cpus": {"$ref": "#/definitions/cpus"}, - "reserve": {"$ref": "#/definitions/optionalMemory"} - } - }, - "backup": { - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": {"type": "boolean"}, - "keepDays": {"type": "integer", "minimum": 1}, - "include": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/definitions/volumeName"}}, - "memory": {"$ref": "#/definitions/memory"}, - "cpus": {"$ref": "#/definitions/cpus"}, - "reserve": {"$ref": "#/definitions/optionalMemory"} - } - }, - "services": { - "type": "object", - "propertyNames": {"pattern": "^[a-z][a-z0-9-]*$", "not": {"enum": ["postgres", "redis", "backup"]}}, - "additionalProperties": {"$ref": "#/definitions/service"} - } - }, - "definitions": { - "duration": {"type": "string", "pattern": "^[0-9]+(s|m|h)$"}, - "cron": {"type": "string", "pattern": "^(@every [0-9]+[smh]|@(hourly|daily|weekly|monthly)|[^ ]+( [^ ]+){4})$"}, - "hostname": {"type": "string", "pattern": "^([a-z0-9]([a-z0-9-]*[a-z0-9])?[.])+project-graphite[.]com$"}, - "memory": {"type": "string", "pattern": "^[0-9]+[MG]$"}, - "optionalMemory": {"type": "string", "pattern": "^([0-9]+[MG])?$"}, - "cpus": {"type": ["string", "number"]}, - "identifier": {"type": "string", "pattern": "^([a-z_][a-z0-9_]*)?$"}, - "volumeName": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$"}, - "strings": {"type": "array", "minItems": 1, "items": {"type": "string"}}, - "names": {"type": "array", "uniqueItems": true, "items": {"type": "string", "pattern": "^[A-Z][A-Z0-9_]*$"}}, - "env": {"type": "object", "additionalProperties": {"type": ["string", "number", "boolean"]}}, - "service": { - "type": "object", - "additionalProperties": false, - "required": ["image"], - "properties": { - "image": {"type": "string", "minLength": 1}, - "tag": {"type": "string", "minLength": 1}, - "entrypoint": {"$ref": "#/definitions/strings"}, - "command": {"$ref": "#/definitions/strings"}, - "port": {"type": "integer", "minimum": 1, "maximum": 65535}, - "public": {"type": "boolean"}, - "hosts": {"type": "array", "uniqueItems": true, "minItems": 1, "items": {"$ref": "#/definitions/hostname"}}, - "jobs": { - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "additionalProperties": false, - "required": ["name", "schedule", "command"], - "properties": { - "name": {"type": "string", "pattern": "^[a-z][a-z0-9-]*$"}, - "schedule": {"$ref": "#/definitions/cron"}, - "command": {"$ref": "#/definitions/strings"}, - "user": {"type": "string", "minLength": 1} - } - } - }, - "env": {"$ref": "#/definitions/env"}, - "secrets": {"$ref": "#/definitions/names"}, - "optional": {"$ref": "#/definitions/names"}, - "dependsOn": {"type": "object", "additionalProperties": {"enum": ["healthy", "started", "completed"]}}, - "volumes": {"type": "array", "uniqueItems": true, "items": {"type": "string", "pattern": "^[^:]+:/[^:]*(:ro)?$"}}, - "files": {"type": "object", "propertyNames": {"pattern": "^/"}, "additionalProperties": {"type": "string"}}, - "health": { - "oneOf": [ - {"type": "string", "pattern": "^/"}, - { - "type": "object", - "additionalProperties": false, - "properties": { - "path": {"type": "string", "pattern": "^/"}, - "test": {"$ref": "#/definitions/strings"}, - "interval": {"$ref": "#/definitions/duration"}, - "timeout": {"$ref": "#/definitions/duration"}, - "retries": {"type": "integer", "minimum": 1}, - "startPeriod": {"$ref": "#/definitions/duration"} - }, - "oneOf": [{"required": ["path"]}, {"required": ["test"]}] - } - ] - }, - "memory": {"$ref": "#/definitions/memory"}, - "cpus": {"$ref": "#/definitions/cpus"}, - "reserve": {"$ref": "#/definitions/memory"}, - "once": {"type": "boolean"}, - "privileged": {"type": "boolean"}, - "pid": {"enum": ["host"]}, - "init": {"type": "boolean"}, - "noNewPrivileges": {"type": "boolean"}, - "stopGracePeriod": {"$ref": "#/definitions/duration"} - } - } - } -} diff --git a/charts/core/values.yaml b/charts/core/values.yaml deleted file mode 100644 index 0c849f0..0000000 --- a/charts/core/values.yaml +++ /dev/null @@ -1,72 +0,0 @@ -# Every option a project can set, with its default. A project writes only what differs, in -# deploy//core.yaml (what runs) and deploy//values.yaml (its settings). - -# Filled in by Deploy from projects/.yml and tags/.json. Never write these. -slug: "" -name: "" -host: "" -tags: {} - -# The project. -enabled: true # false takes the project down -proxied: false # true sends the hostname through Cloudflare's proxy -sleep: {} # after: 30m stops the project after that long without requests and starts - # it on the next one; wake: "0 */6 * * *" (cron or "@every 6h") starts it on a - # schedule for wakeFor (default 15m) - -# Every service under services: starts from these. A service sets: -# image required; the tag comes from tags/.json unless the image names one or tag: is set -# port the port it listens on; with one port in the project, that service gets the hostname -# public true picks the service that gets the hostname when several have a port -# hosts more hostnames for this service, e.g. [api.orbit.project-graphite.com]; needs port -# jobs [{name: nightly, schedule: "0 3 * * *", command: [python, -m, reports]}] runs the -# command inside this running container on a cron schedule (UTC; user: to change -# the user, root by default); not for sleeping projects -# health an HTTP path on port, e.g. /health (checked with wget, then curl), or -# {test: [CMD, ...], interval: 30s, timeout: 5s, retries: 3, startPeriod: 0s} -# env plain settings; ${NAME} and ${NAME:-default} read the secrets file -# secrets names that must be in secrets/.env, passed through as environment -# optional names passed through when present, empty otherwise -# volumes [name:/path] or [name:/path:ro]; named volumes are created -# files {/path/in/container: file-in-deploy//files} -# dependsOn {service: healthy | started | completed}; services also wait for postgres and redis -# command, entrypoint, stopGracePeriod, once (runs to completion), privileged, pid: host, -# init (default true), noNewPrivileges (default true) -# memory, cpus, reserve (default a quarter of memory) -service: - memory: 256M - cpus: "0.5" - -# A PostgreSQL server named postgres, with its data in the postgres-data volume. Every service gets -# DATABASE_URL, POSTGRES_HOST, POSTGRES_PORT, POSTGRES_DB, POSTGRES_USER and POSTGRES_PASSWORD, and -# POSTGRES_PASSWORD must be in the secrets file. -postgres: - image: postgres:18-alpine - database: "" # default: the slug with - as _ - user: "" # default: the database name - settings: {} # server settings, e.g. {shared_buffers: 64MB, max_connections: 40} - env: {} - memory: 256M - cpus: "0.5" - reserve: "" - -# A Redis server named redis, append-only, with its data in the redis-data volume. Every service -# gets REDIS_URL. -redis: - image: redis:8.10-alpine - password: false # true requires REDIS_PASSWORD in the secrets file - evict: true # false keeps every key and refuses writes when full, as a queue needs - maxmemory: "" # default: three quarters of memory - memory: 64M - cpus: "0.25" - reserve: "" - -# A daily pg_dump of postgres into the backups volume, plus a tar of each volume in include. -backup: - keepDays: 14 - include: [] # volume names, e.g. [files-data] - memory: 128M - cpus: "0.25" - reserve: "" - -services: {} diff --git a/deploy/chart.yaml b/deploy/chart.yaml deleted file mode 100644 index 5e8c0e6..0000000 --- a/deploy/chart.yaml +++ /dev/null @@ -1 +0,0 @@ -core: 2.0.0 diff --git a/deploy/console/compose.yaml b/deploy/console/compose.yaml new file mode 100644 index 0000000..45a85a2 --- /dev/null +++ b/deploy/console/compose.yaml @@ -0,0 +1,26 @@ +services: + app: + extends: {file: ../../shared/app.yaml, service: app} + image: ghcr.io/project-graphite/console/app:${IMAGE_TAG:?} + environment: + NODE_ENV: production + SSH_TARGET: ubuntu@135.125.254.59 + GITHUB_CLIENT_ID: ${GITHUB_CLIENT_ID:?} + GITHUB_CLIENT_SECRET: ${GITHUB_CLIENT_SECRET:?} + SESSION_SECRET: ${SESSION_SECRET:?} + ACCESS_TEAM: ${ACCESS_TEAM:?} + ACCESS_AUD: ${ACCESS_AUD:?} + ALERTS_TOKEN: ${ALERTS_TOKEN:-} + SECRETS_KEY: ${SECRETS_KEY:-} + privileged: true + pid: host + volumes: [/var/run/docker.sock:/var/run/docker.sock, data:/data] + healthcheck: + test: [CMD, node, -e, "fetch('http://127.0.0.1:3000/healthz').then((r) => process.exit(r.ok ? 0 : 1), () => process.exit(1))"] + start_period: 10s + +volumes: + data: {labels: {graphite.backup: "true"}} + +networks: + edge: {external: true} diff --git a/deploy/console/core.yaml b/deploy/console/core.yaml deleted file mode 100644 index de9266a..0000000 --- a/deploy/console/core.yaml +++ /dev/null @@ -1,12 +0,0 @@ -proxied: true - -services: - app: - image: ghcr.io/project-graphite/console/app - port: 3000 - privileged: true - pid: host - volumes: [/var/run/docker.sock:/var/run/docker.sock, data:/data] - health: - test: [CMD, node, -e, "fetch('http://127.0.0.1:3000/healthz').then((r) => process.exit(r.ok ? 0 : 1), () => process.exit(1))"] - startPeriod: 10s diff --git a/deploy/console/values b/deploy/console/values new file mode 100644 index 0000000..1d57523 --- /dev/null +++ b/deploy/console/values @@ -0,0 +1,5 @@ +IMAGE_TAG=sha-8d9bb17 +PROXIED=true +APP_MEMORY=192M +APP_CPUS=0.25 +APP_RESERVE=48M diff --git a/deploy/console/values.yaml b/deploy/console/values.yaml deleted file mode 100644 index 2f9bf8d..0000000 --- a/deploy/console/values.yaml +++ /dev/null @@ -1,9 +0,0 @@ -app: - memory: 192M - cpus: "0.25" - env: - NODE_ENV: production - APP_URL: https://console.project-graphite.com - SSH_TARGET: ubuntu@135.125.254.59 - secrets: [GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET, SESSION_SECRET, ACCESS_TEAM, ACCESS_AUD] - optional: [ALERTS_TOKEN] diff --git a/deploy/digital-to-physical/values b/deploy/digital-to-physical/values new file mode 100644 index 0000000..5989882 --- /dev/null +++ b/deploy/digital-to-physical/values @@ -0,0 +1 @@ +IMAGE_TAG=sha-d5ba7b2 diff --git a/deploy/digital-to-physical/values.yaml b/deploy/digital-to-physical/values.yaml deleted file mode 100644 index d4ca941..0000000 --- a/deploy/digital-to-physical/values.yaml +++ /dev/null @@ -1 +0,0 @@ -enabled: true diff --git a/deploy/graphite-hub/values b/deploy/graphite-hub/values new file mode 100644 index 0000000..2b0efa4 --- /dev/null +++ b/deploy/graphite-hub/values @@ -0,0 +1 @@ +IMAGE_TAG=sha-ff5f3e8 diff --git a/deploy/graphite-hub/values.yaml b/deploy/graphite-hub/values.yaml deleted file mode 100644 index d4ca941..0000000 --- a/deploy/graphite-hub/values.yaml +++ /dev/null @@ -1 +0,0 @@ -enabled: true diff --git a/deploy/graphite-message-lab/core.yaml b/deploy/graphite-message-lab/core.yaml deleted file mode 100644 index a7a28fd..0000000 --- a/deploy/graphite-message-lab/core.yaml +++ /dev/null @@ -1,26 +0,0 @@ -sleep: - after: 30m - -postgres: - image: postgres:18.6-trixie -redis: - image: redis:8.10.2-trixie - evict: false - -services: - web: - image: ghcr.io/project-graphite/graphite-message-lab/web - command: [sh, -c, "python manage.py migrate --noinput && exec gunicorn -c python:config.gunicorn config.wsgi"] - port: 8000 - health: {test: [CMD, curl, --fail, "http://localhost:8000/up/databases"], startPeriod: 20s} - stopGracePeriod: 10s - - worker: - image: ghcr.io/project-graphite/graphite-message-lab/worker - command: [celery, -A, config, worker, --loglevel=info, --concurrency=1, --max-tasks-per-child=100] - health: - test: [CMD-SHELL, "celery -A config inspect ping -d celery@$$HOSTNAME -t 10"] - interval: 60s - timeout: 30s - startPeriod: 30s - stopGracePeriod: 30s diff --git a/deploy/graphite-message-lab/values.yaml b/deploy/graphite-message-lab/values.yaml deleted file mode 100644 index c21dbed..0000000 --- a/deploy/graphite-message-lab/values.yaml +++ /dev/null @@ -1,18 +0,0 @@ -web: - memory: 512M - env: &settings - ALLOWED_HOSTS: message-lab.project-graphite.com,localhost - CSRF_TRUSTED_ORIGINS: https://message-lab.project-graphite.com - WEB_CONCURRENCY: "1" - secrets: [SECRET_KEY] - -worker: - memory: 384M - env: *settings - secrets: [SECRET_KEY] - -postgres: - memory: 512M - -redis: - memory: 128M diff --git a/deploy/graphite-tracker/compose.yaml b/deploy/graphite-tracker/compose.yaml new file mode 100644 index 0000000..ce92a27 --- /dev/null +++ b/deploy/graphite-tracker/compose.yaml @@ -0,0 +1,47 @@ +services: + app: + extends: {file: ../../shared/app.yaml, service: app} + image: ghcr.io/project-graphite/graphite-tracker/app:${IMAGE_TAG:?} + environment: + AUTH_TRUSTED_ORIGINS: https://${HOST} + DATABASE_URL: postgresql://graphite:${POSTGRES_PASSWORD:?}@postgres:5432/graphite_tracker + POSTGRES_HOST: postgres + POSTGRES_PORT: "5432" + POSTGRES_DB: graphite_tracker + POSTGRES_USER: graphite + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?} + REDIS_URL: redis://redis:6379 + AUTH_ACCESS_TOKEN_SECRET: ${AUTH_ACCESS_TOKEN_SECRET:?} + GAME_SOURCE: ${GAME_SOURCE:-igdb} + DISABLED_SOURCES: ${DISABLED_SOURCES:-} + TMDB_READ_ACCESS_TOKEN: ${TMDB_READ_ACCESS_TOKEN:-} + IGDB_CLIENT_ID: ${IGDB_CLIENT_ID:-} + IGDB_CLIENT_SECRET: ${IGDB_CLIENT_SECRET:-} + RAWG_API_KEY: ${RAWG_API_KEY:-} + EMAIL_HOST: ${EMAIL_HOST:?} + EMAIL_PORT: ${EMAIL_PORT:-587} + DEFAULT_FROM_EMAIL: ${DEFAULT_FROM_EMAIL:?} + EMAIL_HOST_USER: ${EMAIL_HOST_USER:-} + EMAIL_HOST_PASSWORD: ${EMAIL_HOST_PASSWORD:-} + healthcheck: + test: [CMD-SHELL, "wget -q -O /dev/null http://127.0.0.1:3000/api/v1/health || curl -fsS -o /dev/null http://127.0.0.1:3000/api/v1/health"] + start_period: 20s + depends_on: + postgres: {condition: service_healthy} + redis: {condition: service_healthy} + + postgres: + extends: {file: ../../shared/postgres.yaml, service: postgres} + environment: + POSTGRES_DB: graphite_tracker + POSTGRES_USER: graphite + + redis: + extends: {file: ../../shared/redis.yaml, service: redis} + +volumes: + postgres-data: {} + redis-data: {} + +networks: + edge: {external: true} diff --git a/deploy/graphite-tracker/core.yaml b/deploy/graphite-tracker/core.yaml deleted file mode 100644 index 8ec8645..0000000 --- a/deploy/graphite-tracker/core.yaml +++ /dev/null @@ -1,10 +0,0 @@ -postgres: - database: graphite_tracker - user: graphite -redis: {} - -services: - app: - image: ghcr.io/project-graphite/graphite-tracker/app - port: 3000 - health: {path: /api/v1/health, startPeriod: 20s} diff --git a/deploy/graphite-tracker/values b/deploy/graphite-tracker/values new file mode 100644 index 0000000..adf12cf --- /dev/null +++ b/deploy/graphite-tracker/values @@ -0,0 +1,10 @@ +IMAGE_TAG=sha-7533f37 +APP_MEMORY=512M +APP_CPUS=0.75 +APP_RESERVE=128M +POSTGRES_MEMORY=384M +POSTGRES_CPUS=0.75 +POSTGRES_RESERVE=96M +REDIS_MEMORY=96M +REDIS_RESERVE=24M +REDIS_MAXMEMORY=64mb diff --git a/deploy/graphite-tracker/values.yaml b/deploy/graphite-tracker/values.yaml deleted file mode 100644 index 313e9d1..0000000 --- a/deploy/graphite-tracker/values.yaml +++ /dev/null @@ -1,18 +0,0 @@ -app: - memory: 512M - cpus: "0.75" - env: - APP_URL: https://graphite-tracker.project-graphite.com - AUTH_TRUSTED_ORIGINS: https://graphite-tracker.project-graphite.com - GAME_SOURCE: ${GAME_SOURCE:-igdb} - EMAIL_PORT: ${EMAIL_PORT:-587} - secrets: [AUTH_ACCESS_TOKEN_SECRET, EMAIL_HOST, DEFAULT_FROM_EMAIL] - optional: [TMDB_READ_ACCESS_TOKEN, IGDB_CLIENT_ID, IGDB_CLIENT_SECRET, RAWG_API_KEY, DISABLED_SOURCES, EMAIL_HOST_USER, EMAIL_HOST_PASSWORD] - -postgres: - memory: 384M - cpus: "0.75" - -redis: - memory: 96M - maxmemory: 64mb diff --git a/deploy/pixelpocket/compose.yaml b/deploy/pixelpocket/compose.yaml new file mode 100644 index 0000000..f90876c --- /dev/null +++ b/deploy/pixelpocket/compose.yaml @@ -0,0 +1,100 @@ +services: + frontend: + extends: {file: ../../shared/app.yaml, service: app} + image: ghcr.io/project-graphite/pixelpocket/frontend:${IMAGE_TAG:?} + healthcheck: + test: [CMD-SHELL, "wget -q -O /dev/null http://127.0.0.1:80/ || curl -fsS -o /dev/null http://127.0.0.1:80/"] + start_period: 10s + depends_on: + backend: {condition: service_healthy} + + backend: + extends: {file: ../../shared/app.yaml, service: service} + image: ghcr.io/project-graphite/pixelpocket/backend:${IMAGE_TAG:?} + expose: ["5000"] + environment: + NODE_ENV: production + PORT: "5000" + DATABASE_LOCAL: mongodb://pixelpocket:${MONGO_PASSWORD:?}@mongo:27017/pixelpocket?authSource=admin + JWT_SECRET: ${JWT_SECRET:?} + JWT_EXPIRES_IN: ${JWT_EXPIRES_IN:-90d} + JWT_COOKIE_EXPIRES_IN: ${JWT_COOKIE_EXPIRES_IN:-90} + FRONTEND_URL: https://${HOST} + EMAIL_HOST: ${EMAIL_HOST:?} + EMAIL_PORT: ${EMAIL_PORT:-587} + EMAIL_FROM: ${EMAIL_FROM:?} + EMAIL_USERNAME: ${EMAIL_USERNAME:-} + EMAIL_PASSWORD: ${EMAIL_PASSWORD:-} + S3_ENDPOINT: http://minio:9000 + S3_BUCKET: pixelpocket + S3_ACCESS_KEY: pixelpocket-app + S3_SECRET_KEY: ${S3_SECRET_KEY:?} + healthcheck: + test: [CMD-SHELL, "wget -q -O /dev/null http://127.0.0.1:5000/api/v1/users/session || curl -fsS -o /dev/null http://127.0.0.1:5000/api/v1/users/session"] + start_period: 20s + depends_on: + mongo: {condition: service_healthy} + minio-init: {condition: service_completed_successfully} + + mongo: + extends: {file: ../../shared/app.yaml, service: service} + image: mongo:7 + environment: + MONGO_INITDB_ROOT_USERNAME: pixelpocket + MONGO_INITDB_ROOT_PASSWORD: ${MONGO_PASSWORD:?} + volumes: [mongo-data:/data/db] + healthcheck: + test: [CMD-SHELL, "mongosh -u \"$$MONGO_INITDB_ROOT_USERNAME\" -p \"$$MONGO_INITDB_ROOT_PASSWORD\" --authenticationDatabase admin --quiet --eval \"db.adminCommand('ping')\""] + interval: 10s + retries: 10 + deploy: + resources: + limits: {memory: 512M} + reservations: {memory: 128M} + + minio: + extends: {file: ../../shared/app.yaml, service: service} + image: chainguard/minio@sha256:25bc63a89475bc494bfbce9bbeddb0421aece3f3968d77c809db5104d07b4545 + command: [server, /data] + environment: + MINIO_ROOT_USER: pixelpocket-admin + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?} + volumes: [minio-data:/data] + configs: + - {source: minio-healthcheck, target: /scripts/healthcheck.sh} + healthcheck: + test: [CMD, bash, /scripts/healthcheck.sh] + interval: 10s + retries: 10 + + minio-init: + extends: {file: ../../shared/app.yaml, service: service} + image: chainguard/minio@sha256:25bc63a89475bc494bfbce9bbeddb0421aece3f3968d77c809db5104d07b4545 + entrypoint: [bash, /scripts/init.sh] + restart: "no" + environment: + MINIO_ROOT_USER: pixelpocket-admin + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?} + S3_BUCKET: pixelpocket + S3_ACCESS_KEY: pixelpocket-app + S3_SECRET_KEY: ${S3_SECRET_KEY:?} + configs: + - {source: minio-init, target: /scripts/init.sh} + healthcheck: !reset {} + depends_on: + minio: {condition: service_healthy} + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + reservations: !reset {} + +volumes: + mongo-data: {} + minio-data: {} + +configs: + minio-healthcheck: {file: ./files/minio-healthcheck.sh} + minio-init: {file: ./files/minio-init.sh} + +networks: + edge: {external: true} diff --git a/deploy/pixelpocket/core.yaml b/deploy/pixelpocket/core.yaml deleted file mode 100644 index 619204a..0000000 --- a/deploy/pixelpocket/core.yaml +++ /dev/null @@ -1,35 +0,0 @@ -services: - frontend: - image: ghcr.io/project-graphite/pixelpocket/frontend - port: 80 - public: true - dependsOn: {backend: healthy} - health: {path: /, startPeriod: 10s} - - backend: - image: ghcr.io/project-graphite/pixelpocket/backend - port: 5000 - dependsOn: {mongo: healthy, minio-init: completed} - health: {path: /api/v1/users/session, startPeriod: 20s} - - mongo: - image: mongo:7 - volumes: [mongo-data:/data/db] - health: - test: [CMD-SHELL, "mongosh -u \"$$MONGO_INITDB_ROOT_USERNAME\" -p \"$$MONGO_INITDB_ROOT_PASSWORD\" --authenticationDatabase admin --quiet --eval \"db.adminCommand('ping')\""] - interval: 10s - retries: 10 - - minio: - image: chainguard/minio@sha256:25bc63a89475bc494bfbce9bbeddb0421aece3f3968d77c809db5104d07b4545 - command: [server, /data] - volumes: [minio-data:/data] - files: {/scripts/healthcheck.sh: minio-healthcheck.sh} - health: {test: [CMD, bash, /scripts/healthcheck.sh], interval: 10s, retries: 10} - - minio-init: - image: chainguard/minio@sha256:25bc63a89475bc494bfbce9bbeddb0421aece3f3968d77c809db5104d07b4545 - entrypoint: [bash, /scripts/init.sh] - once: true - dependsOn: {minio: healthy} - files: {/scripts/init.sh: minio-init.sh} diff --git a/deploy/pixelpocket/values b/deploy/pixelpocket/values new file mode 100644 index 0000000..1a94ea6 --- /dev/null +++ b/deploy/pixelpocket/values @@ -0,0 +1,5 @@ +IMAGE_TAG=sha-e38c8e1 +APP_PORT=80 +APP_MEMORY=64M +APP_CPUS=0.25 +APP_RESERVE=16M diff --git a/deploy/pixelpocket/values.yaml b/deploy/pixelpocket/values.yaml deleted file mode 100644 index f05343e..0000000 --- a/deploy/pixelpocket/values.yaml +++ /dev/null @@ -1,41 +0,0 @@ -frontend: - memory: 64M - cpus: "0.25" - -backend: - memory: 256M - env: - NODE_ENV: production - PORT: "5000" - DATABASE_LOCAL: mongodb://pixelpocket:${MONGO_PASSWORD}@mongo:27017/pixelpocket?authSource=admin - JWT_EXPIRES_IN: ${JWT_EXPIRES_IN:-90d} - JWT_COOKIE_EXPIRES_IN: ${JWT_COOKIE_EXPIRES_IN:-90} - FRONTEND_URL: https://pixelpocket.project-graphite.com - EMAIL_PORT: ${EMAIL_PORT:-587} - S3_ENDPOINT: http://minio:9000 - S3_BUCKET: pixelpocket - S3_ACCESS_KEY: pixelpocket-app - secrets: [JWT_SECRET, EMAIL_HOST, EMAIL_FROM, S3_SECRET_KEY] - optional: [EMAIL_USERNAME, EMAIL_PASSWORD] - -mongo: - memory: 512M - env: - MONGO_INITDB_ROOT_USERNAME: pixelpocket - MONGO_INITDB_ROOT_PASSWORD: ${MONGO_PASSWORD} - secrets: [MONGO_PASSWORD] - -minio: - memory: 256M - env: - MINIO_ROOT_USER: pixelpocket-admin - secrets: [MINIO_ROOT_PASSWORD] - -minio-init: - memory: 128M - cpus: "0.25" - env: - MINIO_ROOT_USER: pixelpocket-admin - S3_BUCKET: pixelpocket - S3_ACCESS_KEY: pixelpocket-app - secrets: [MINIO_ROOT_PASSWORD, S3_SECRET_KEY] diff --git a/deploy/strata/compose.yaml b/deploy/strata/compose.yaml new file mode 100644 index 0000000..d53bdd0 --- /dev/null +++ b/deploy/strata/compose.yaml @@ -0,0 +1,45 @@ +services: + app: + extends: {file: ../../shared/app.yaml, service: app} + image: ghcr.io/project-graphite/strata/app:${IMAGE_TAG:?} + environment: + NODE_ENV: production + AUTH_TRUSTED_ORIGINS: https://${HOST} + DATABASE_URL: postgresql://${PROJECT}:${POSTGRES_PASSWORD:?}@postgres:5432/${PROJECT} + POSTGRES_HOST: postgres + POSTGRES_PORT: "5432" + POSTGRES_DB: ${PROJECT} + POSTGRES_USER: ${PROJECT} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?} + REDIS_URL: redis://:${REDIS_PASSWORD:?}@redis:6379 + AUTH_ACCESS_TOKEN_SECRET: ${AUTH_ACCESS_TOKEN_SECRET:?} + DATA_ENCRYPTION_KEY: ${DATA_ENCRYPTION_KEY:?} + EMAIL_HOST: ${EMAIL_HOST:?} + EMAIL_PORT: ${EMAIL_PORT:-587} + DEFAULT_FROM_EMAIL: ${DEFAULT_FROM_EMAIL:?} + EMAIL_HOST_USER: ${EMAIL_HOST_USER:-} + EMAIL_HOST_PASSWORD: ${EMAIL_HOST_PASSWORD:-} + VAPID_PUBLIC_KEY: ${VAPID_PUBLIC_KEY:-} + VAPID_PRIVATE_KEY: ${VAPID_PRIVATE_KEY:-} + VAPID_SUBJECT: ${VAPID_SUBJECT:-} + volumes: [files-data:/data/files] + healthcheck: + test: [CMD-SHELL, "wget -q -O /dev/null http://127.0.0.1:3000/api/v1/health || curl -fsS -o /dev/null http://127.0.0.1:3000/api/v1/health"] + start_period: 20s + depends_on: + postgres: {condition: service_healthy} + redis: {condition: service_healthy} + + postgres: + extends: {file: ../../shared/postgres.yaml, service: postgres} + + redis: + extends: {file: ../../shared/redis-password.yaml, service: redis} + +volumes: + files-data: {labels: {graphite.backup: "true"}} + postgres-data: {} + redis-data: {} + +networks: + edge: {external: true} diff --git a/deploy/strata/core.yaml b/deploy/strata/core.yaml deleted file mode 100644 index d78c6c6..0000000 --- a/deploy/strata/core.yaml +++ /dev/null @@ -1,13 +0,0 @@ -postgres: - settings: {shared_buffers: 64MB, work_mem: 4MB, max_connections: 40} -redis: - password: true -backup: - include: [files-data] - -services: - app: - image: ghcr.io/project-graphite/strata/app - port: 3000 - health: {path: /api/v1/health, startPeriod: 20s} - volumes: [files-data:/data/files] diff --git a/deploy/strata/values b/deploy/strata/values new file mode 100644 index 0000000..706aa8c --- /dev/null +++ b/deploy/strata/values @@ -0,0 +1,8 @@ +IMAGE_TAG=sha-ebcde55 +APP_MEMORY=512M +APP_CPUS=0.75 +APP_RESERVE=128M +POSTGRES_MEMORY=320M +POSTGRES_RESERVE=80M +POSTGRES_SHARED_BUFFERS=64MB +POSTGRES_MAX_CONNECTIONS=40 diff --git a/deploy/strata/values.yaml b/deploy/strata/values.yaml deleted file mode 100644 index ad03597..0000000 --- a/deploy/strata/values.yaml +++ /dev/null @@ -1,13 +0,0 @@ -app: - memory: 512M - cpus: "0.75" - env: - NODE_ENV: production - APP_URL: https://strata.project-graphite.com - AUTH_TRUSTED_ORIGINS: https://strata.project-graphite.com - EMAIL_PORT: ${EMAIL_PORT:-587} - secrets: [AUTH_ACCESS_TOKEN_SECRET, DATA_ENCRYPTION_KEY, EMAIL_HOST, DEFAULT_FROM_EMAIL] - optional: [EMAIL_HOST_USER, EMAIL_HOST_PASSWORD, VAPID_PUBLIC_KEY, VAPID_PRIVATE_KEY, VAPID_SUBJECT] - -postgres: - memory: 320M diff --git a/deploy/traceboard/values.yaml b/deploy/traceboard/values.yaml deleted file mode 100644 index bc11441..0000000 --- a/deploy/traceboard/values.yaml +++ /dev/null @@ -1 +0,0 @@ -enabled: false diff --git a/edge/backup.py b/edge/backup.py new file mode 100644 index 0000000..e67126d --- /dev/null +++ b/edge/backup.py @@ -0,0 +1,403 @@ +import contextlib +import datetime +import hashlib +import http.client +import io +import json +import os +import pathlib +import shutil +import socket +import sqlite3 +import sys +import tarfile +import tempfile +import time +import traceback +import urllib.parse + +ROOT = pathlib.Path(os.environ.get("BACKUP_ROOT", "/backups")) +VOLUMES = pathlib.Path(os.environ.get("VOLUMES_ROOT", "/volumes")) +REQUESTS = ROOT / ".requests" +WORK = ROOT / ".work" +KEEP_DAYS = int(os.environ.get("KEEP_DAYS", "14")) +HOUR = int(os.environ.get("BACKUP_HOUR", "3")) +STAMP = "%Y%m%dT%H%M%SZ" +SQLITE = b"SQLite format 3\x00" +SIDECARS = ("-journal", "-wal", "-shm") +DUMP = "/tmp/graphite-backup.dump" + + +def log(message): + print(f"{datetime.datetime.now(datetime.UTC):%Y-%m-%dT%H:%M:%SZ} {message}", flush=True) + + +class Docker(http.client.HTTPConnection): + def __init__(self): + super().__init__("localhost", timeout=1800) + + def connect(self): + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.sock.settimeout(self.timeout) + self.sock.connect("/var/run/docker.sock") + + +def docker(method, path, body=None, stream=False, headers=None): + connection = Docker() + if isinstance(body, (dict, list)): + body, headers = json.dumps(body), {"Content-Type": "application/json"} + connection.request(method, path, body=body, headers=headers or {}) + response = connection.getresponse() + if response.status >= 400: + raise RuntimeError(f"Docker answered {response.status} to {method} {path.split('?')[0]}: {response.read().decode(errors='replace')[:300]}") + if stream: + return response + data = response.read() + connection.close() + return json.loads(data) if data else None + + +def filtered(path, **filters): + return f"{path}?all=1&filters={urllib.parse.quote(json.dumps(filters))}" + + +def execute(container, command): + created = docker("POST", f"/containers/{container}/exec", {"Cmd": command, "AttachStdout": True, "AttachStderr": True}) + response = docker("POST", f"/exec/{created['Id']}/start", {"Detach": False, "Tty": False}, stream=True) + output = {1: bytearray(), 2: bytearray()} + while header := response.read(8): + output.setdefault(header[0], bytearray()).extend(response.read(int.from_bytes(header[4:], "big"))) + code = docker("GET", f"/exec/{created['Id']}/json")["ExitCode"] + if code: + raise RuntimeError(f"{command[0]} exited with {code}: {output[2].decode(errors='replace')[-500:].strip()}") + return output[1].decode() + + +def digest(stream): + hasher = hashlib.sha256() + while chunk := stream.read(1 << 20): + hasher.update(chunk) + return hasher.hexdigest() + + +def file_digest(path): + with open(path, "rb") as stream: + return digest(stream) + + +def is_sqlite(path): + with open(path, "rb") as stream: + return stream.read(len(SQLITE)) == SQLITE + + +def copy_sqlite(source, target): + target.unlink(missing_ok=True) + with contextlib.closing(sqlite3.connect(f"{source.as_uri()}?mode=ro", uri=True)) as reader: + with contextlib.closing(sqlite3.connect(target)) as writer: + reader.backup(writer) + + +def integrity(path): + with contextlib.closing(sqlite3.connect(f"{path.as_uri()}?mode=ro", uri=True)) as database: + result = database.execute("PRAGMA integrity_check").fetchone()[0] + if result != "ok": + raise RuntimeError(f"integrity_check answered {result}") + + +def sidecar(path): + for suffix in SIDECARS: + if path.name.endswith(suffix): + base = path.with_name(path.name[: -len(suffix)]) + return base.is_file() and is_sqlite(base) + return False + + +def pack_volume(source, target, work): + databases = [] + with tarfile.open(target, "w:gz") as tar: + for path in sorted(source.rglob("*")): + name = path.relative_to(source).as_posix() + if path.is_file() and not path.is_symlink() and not sidecar(path) and is_sqlite(path): + copy = work / "copy.db" + copy_sqlite(path, copy) + info = tar.gettarinfo(path, name) + info.size = copy.stat().st_size + with open(copy, "rb") as stream: + tar.addfile(info, stream) + databases.append(name) + elif not (path.is_file() and sidecar(path)): + tar.add(path, name, recursive=False) + return databases + + +def check_volume(packed, item, work): + with tarfile.open(fileobj=packed, mode="r|gz") as inner: + for entry in inner: + if not entry.isfile(): + continue + with inner.extractfile(entry) as data: + if entry.name not in item["sqlite"]: + digest(data) + continue + copy = work / "check.db" + with open(copy, "wb") as out: + shutil.copyfileobj(data, out) + integrity(copy) + + +def verify(archive, project): + errors = [] + with tempfile.TemporaryDirectory(dir=WORK) as work, tarfile.open(archive) as tar: + manifest = json.load(tar.extractfile("manifest.json")) + if manifest.get("project") != project: + return [f"the archive is for {manifest.get('project')}, not {project}"] + for item in manifest["items"]: + try: + with tar.extractfile(item["path"]) as stream: + if digest(stream) != item["sha256"]: + errors.append(f"{item['path']} does not match its checksum") + continue + if item["kind"] == "volume": + with tar.extractfile(item["path"]) as packed: + check_volume(packed, item, pathlib.Path(work)) + except (KeyError, OSError, tarfile.TarError, sqlite3.Error, RuntimeError) as error: + errors.append(f"{item['path']}: {error}") + return errors + + +def dump_postgres(container, target): + if container["State"] != "running": + raise RuntimeError("its container is not running") + script = f'set -e; pg_dump -Fc -U "$POSTGRES_USER" -d "$POSTGRES_DB" -f {DUMP}; pg_restore --list {DUMP} > /dev/null; sha256sum {DUMP}' + try: + expected = execute(container["Id"], ["sh", "-c", script]).split()[0] + with tarfile.open(fileobj=docker("GET", f"/containers/{container['Id']}/archive?path={DUMP}", stream=True), mode="r|") as stream: + for member in stream: + with stream.extractfile(member) as source, open(target, "wb") as out: + shutil.copyfileobj(source, out) + finally: + execute(container["Id"], ["rm", "-f", DUMP]) + if file_digest(target) != expected: + raise RuntimeError("the dump changed on its way out of the container") + + +def targets(): + found = {} + for container in docker("GET", filtered("/containers/json", label=["graphite.backup.postgres=true"])): + project = container["Labels"].get("com.docker.compose.project", "other") + found.setdefault(project, {"postgres": [], "volumes": []})["postgres"].append(container) + for volume in docker("GET", filtered("/volumes", label=["graphite.backup=true"]))["Volumes"] or []: + project = (volume.get("Labels") or {}).get("com.docker.compose.project", "other") + found.setdefault(project, {"postgres": [], "volumes": []})["volumes"].append(volume) + return found + + +def write_summary(folder, stamp, summary): + temporary = folder / f"{stamp}.json.partial" + temporary.write_text(json.dumps(summary, indent=2), encoding="utf-8") + temporary.replace(folder / f"{stamp}.json") + + +def summarise(archive, project, manifest, errors): + errors = errors + verify(archive, project) + return {**manifest, "archive": archive.name, "bytes": archive.stat().st_size, "sha256": file_digest(archive), "ok": not errors, "errors": errors} + + +def backup(project, found, now): + stamp = now.strftime(STAMP) + folder = ROOT / project + folder.mkdir(parents=True, exist_ok=True) + archive = folder / f"{stamp}.tar" + manifest = {"project": project, "stamp": stamp, "started": now.isoformat(), "items": []} + errors = [] + with tempfile.TemporaryDirectory(dir=WORK) as work, tarfile.open(f"{archive}.partial", "w") as tar: + work = pathlib.Path(work) + + def add(path, name, **item): + tar.add(path, name) + manifest["items"].append({"path": name, "bytes": path.stat().st_size, "sha256": file_digest(path), **item}) + + for container in found["postgres"]: + service = container["Labels"].get("com.docker.compose.service", container["Id"][:12]) + try: + dump_postgres(container, work / f"{service}.dump") + add(work / f"{service}.dump", f"postgres/{service}.dump", kind="postgres", service=service) + except (RuntimeError, OSError, tarfile.TarError) as error: + errors.append(f"postgres {service}: {error}") + for volume in found["volumes"]: + name = (volume.get("Labels") or {}).get("com.docker.compose.volume", volume["Name"]) + try: + databases = pack_volume(VOLUMES / volume["Name"] / "_data", work / f"{name}.tar.gz", work) + add(work / f"{name}.tar.gz", f"volumes/{name}.tar.gz", kind="volume", volume=volume["Name"], sqlite=databases) + except (OSError, sqlite3.Error, tarfile.TarError) as error: + errors.append(f"volume {name}: {error}") + manifest["finished"] = datetime.datetime.now(datetime.UTC).isoformat() + data = json.dumps(manifest, indent=2).encode() + info = tarfile.TarInfo("manifest.json") + info.size, info.mtime = len(data), int(time.time()) + tar.addfile(info, io.BytesIO(data)) + pathlib.Path(f"{archive}.partial").replace(archive) + summary = summarise(archive, project, manifest, errors) + write_summary(folder, stamp, summary) + log(f"{project}: {archive.name}, {summary['bytes']} bytes, {'verified' if summary['ok'] else 'FAILED: ' + '; '.join(summary['errors'])}") + return summary + + +def prune(now): + cutoff = now - datetime.timedelta(days=KEEP_DAYS) + for path in ROOT.glob("*/*.tar"): + try: + stamp = datetime.datetime.strptime(path.stem, STAMP).replace(tzinfo=datetime.UTC) + except ValueError: + continue + if stamp < cutoff: + path.unlink() + path.with_suffix(".json").unlink(missing_ok=True) + log(f"removed {path.parent.name}/{path.name}") + for path in REQUESTS.glob("*.result"): + if path.stat().st_mtime < cutoff.timestamp(): + path.unlink() + + +def run(only=None): + now = datetime.datetime.now(datetime.UTC) + found = targets() + if only is not None and only not in found: + raise RuntimeError(f"{only} has no container labelled graphite.backup.postgres and no volume labelled graphite.backup") + summaries = [] + for project, items in sorted(found.items()): + if only is not None and project != only: + continue + try: + summaries.append(backup(project, items, now)) + except (OSError, RuntimeError, tarfile.TarError, json.JSONDecodeError) as error: + (ROOT / project).mkdir(parents=True, exist_ok=True) + write_summary(ROOT / project, now.strftime(STAMP), {"project": project, "stamp": now.strftime(STAMP), "started": now.isoformat(), "items": [], "ok": False, "errors": [str(error)]}) + log(f"{project}: FAILED: {error}") + summaries.append({"project": project, "ok": False, "errors": [str(error)]}) + prune(now) + return summaries + + +def project_containers(project): + return docker("GET", filtered("/containers/json", label=[f"com.docker.compose.project={project}"])) + + +def put_file(container, source, size, name, work): + bundle = work / "upload.tar" + with tarfile.open(bundle, "w") as tar: + info = tarfile.TarInfo(name) + info.size, info.mtime = size, int(time.time()) + tar.addfile(info, source) + with open(bundle, "rb") as body: + docker("PUT", f"/containers/{container}/archive?path=/tmp", body=body, headers={"Content-Type": "application/x-tar", "Content-Length": str(bundle.stat().st_size)}) + + +def wait_healthy(ids, timeout=300): + deadline = time.time() + timeout + while time.time() < deadline: + states = [docker("GET", f"/containers/{container}/json")["State"] for container in ids] + if all(state["Running"] and state.get("Health", {}).get("Status", "healthy") == "healthy" for state in states): + return + time.sleep(5) + raise RuntimeError("the project was not healthy five minutes after the restore") + + +def restore(project, name): + archive = ROOT / project / name + errors = verify(archive, project) + if errors: + raise RuntimeError("the archive failed verification: " + "; ".join(errors)) + containers = project_containers(project) + databases = {c["Labels"].get("com.docker.compose.service"): c for c in containers if c["Labels"].get("graphite.backup.postgres") == "true"} + running = [c["Id"] for c in containers if c["State"] == "running" and c["Id"] not in {d["Id"] for d in databases.values()}] + with tempfile.TemporaryDirectory(dir=WORK) as work, tarfile.open(archive) as tar: + work = pathlib.Path(work) + manifest = json.load(tar.extractfile("manifest.json")) + dumps = [item for item in manifest["items"] if item["kind"] == "postgres"] + for item in dumps: + database = databases.get(item["service"]) + if not database or database["State"] != "running": + raise RuntimeError(f"{project} has no running {item['service']} to restore into") + with tar.extractfile(item["path"]) as source: + put_file(database["Id"], source, item["bytes"], "graphite-restore.dump", work) + execute(database["Id"], ["pg_restore", "--list", "/tmp/graphite-restore.dump"]) + for container in running: + docker("POST", f"/containers/{container}/stop?t=30") + try: + for item in dumps: + execute(databases[item["service"]]["Id"], [ + "sh", "-c", + 'pg_restore --clean --if-exists --no-owner --single-transaction -U "$POSTGRES_USER" -d "$POSTGRES_DB" /tmp/graphite-restore.dump', + ]) + for item in (entry for entry in manifest["items"] if entry["kind"] == "volume"): + target = VOLUMES / item["volume"] / "_data" + for child in target.iterdir(): + if child.is_dir() and not child.is_symlink(): + shutil.rmtree(child) + else: + child.unlink() + with tar.extractfile(item["path"]) as packed, tarfile.open(fileobj=packed, mode="r|gz") as inner: + inner.extractall(target, filter="tar") + finally: + for database in databases.values(): + execute(database["Id"], ["rm", "-f", "/tmp/graphite-restore.dump"]) + for container in running: + docker("POST", f"/containers/{container}/start") + wait_healthy([*running, *(d["Id"] for d in databases.values())]) + return f"Restored {project} from {name}: {len(dumps)} databases and {len(manifest['items']) - len(dumps)} volumes; it is healthy." + + +def handle(request): + working = request.with_suffix(".running") + request.replace(working) + order = json.loads(working.read_text(encoding="utf-8")) + log(f"{order['action']} {order.get('project')} {order.get('archive', '')} for {order.get('by')}") + result = {**order, "at": datetime.datetime.now(datetime.UTC).isoformat()} + try: + if order["action"] == "run": + summaries = run(order["project"]) + result.update(ok=all(entry["ok"] for entry in summaries), output="; ".join(f"{entry['project']}: {'verified' if entry['ok'] else '; '.join(entry['errors'])}" for entry in summaries)) + elif order["action"] == "verify": + archive = ROOT / order["project"] / order["archive"] + with tarfile.open(archive) as tar: + manifest = json.load(tar.extractfile("manifest.json")) + summary = summarise(archive, order["project"], {**manifest, "stamp": archive.stem, "imported": True}, []) + write_summary(archive.parent, archive.stem, summary) + result.update(ok=summary["ok"], output="verified" if summary["ok"] else "; ".join(summary["errors"])) + elif order["action"] == "restore": + result.update(ok=True, output=restore(order["project"], order["archive"])) + else: + result.update(ok=False, output=f"unknown action {order['action']}") + except Exception as error: + traceback.print_exc() + result.update(ok=False, output=str(error)) + working.with_suffix(".result").write_text(json.dumps(result), encoding="utf-8") + working.unlink() + log(f"{order['action']} {order.get('project')}: {'done' if result['ok'] else 'FAILED: ' + result['output']}") + + +def next_nightly(now): + today = now.replace(hour=HOUR, minute=0, second=0, microsecond=0) + return today if now < today else today + datetime.timedelta(days=1) + + +def serve(): + for folder in (REQUESTS, WORK): + folder.mkdir(parents=True, exist_ok=True) + for interrupted in REQUESTS.glob("*.running"): + interrupted.with_suffix(".result").write_text(json.dumps({**json.loads(interrupted.read_text(encoding="utf-8")), "ok": False, "output": "interrupted by a restart"}), encoding="utf-8") + interrupted.unlink() + due = next_nightly(datetime.datetime.now(datetime.UTC)) + log(f"waiting for requests; the nightly run is at {due:%Y-%m-%d %H:%M}Z") + while True: + for request in sorted(REQUESTS.glob("*.request")): + handle(request) + if datetime.datetime.now(datetime.UTC) >= due: + run() + due = next_nightly(datetime.datetime.now(datetime.UTC)) + time.sleep(5) + + +if __name__ == "__main__": + sys.exit(serve()) diff --git a/edge/backup.yaml b/edge/backup.yaml new file mode 100644 index 0000000..6e5f42f --- /dev/null +++ b/edge/backup.yaml @@ -0,0 +1,23 @@ +services: + backup: + image: python:3.14-alpine + command: [python, /app/backup.py] + environment: + KEEP_DAYS: "14" + BACKUP_HOUR: "3" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /var/lib/docker/volumes:/volumes + - /var/lib/graphite/backups:/backups + - ./files/backup.py:/app/backup.py:ro + restart: unless-stopped + logging: + driver: json-file + options: {max-size: 10m, max-file: "5"} + security_opt: + - no-new-privileges:true + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M diff --git a/edge/wake.yaml b/edge/wake.yaml deleted file mode 100644 index fd5769e..0000000 --- a/edge/wake.yaml +++ /dev/null @@ -1,18 +0,0 @@ -name: wake - -services: - ofelia: - image: mcuadros/ofelia:0.3.22 - command: [daemon, --config=/etc/ofelia.ini] - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - - ./ofelia.ini:/etc/ofelia.ini:ro - restart: unless-stopped - logging: - driver: json-file - options: {max-size: 10m, max-file: "5"} - deploy: - resources: - limits: - cpus: "0.1" - memory: 32M diff --git a/host/agent.py b/host/agent.py index baeaa54..7ba86b5 100644 --- a/host/agent.py +++ b/host/agent.py @@ -10,8 +10,10 @@ BASE = pathlib.Path(os.environ.get("GRAPHITE_HOME", "/srv/graphite")) TOKEN = os.environ["DEPLOY_TOKEN"].encode() +CONSOLE_TOKEN = os.environ.get("CONSOLE_TOKEN", "").encode() +ENV = pathlib.Path("/etc/graphite") NAME = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,38}[a-z0-9])?$") -FILES = {"compose.yaml", "traefik.yml", "ofelia.ini", "sleep"} +FILES = re.compile(r"^(compose\.yaml|traefik\.yml|values|files/[A-Za-z0-9._-]+)$") SKEW = 300 @@ -38,28 +40,36 @@ def run(*command): return output +def env_file(name): + path = ENV / f"{name}.env" + return path if path.exists() else BASE / name / "secrets.env" + + def keys(path): if not path.exists(): return set() return {line.split("=", 1)[0] for line in path.read_text(encoding="utf-8").splitlines() if "=" in line} +def compose_options(name): + values = BASE / name / "values" + return [*(["--env-file", str(values)] if values.exists() else []), "--env-file", str(env_file(name))] + + def up(name, request): folder = BASE / name - unknown = sorted(set(request.get("files", {})) - FILES) + unknown = sorted(file for file in request.get("files", {}) if not FILES.match(file)) if unknown: raise ValueError(f"{', '.join(unknown)} is not a file the agent writes") + missing = sorted(set(request.get("require", [])) - keys(env_file(name))) + if missing: + raise LookupError(f"{name} is waiting on secrets: {', '.join(missing)}") for file, content in request.get("files", {}).items(): if content is None: (folder / file).unlink(missing_ok=True) else: write(folder / file, content) - missing = sorted(set(request.get("require", [])) - keys(folder / "secrets.env")) - if missing: - raise LookupError(f"{name} is waiting on secrets: {', '.join(missing)}") output = run(str(BASE / "bin" / "graphite-up"), name) - if request.get("restart"): - output += run("docker", "compose", "--project-directory", str(folder), "-p", name, "restart") if "route" in request: route = BASE / "edge" / "routes" / f"{name}.yml" if request["route"] is None: @@ -72,13 +82,15 @@ def up(name, request): def down(name): folder = BASE / name (BASE / "edge" / "routes" / f"{name}.yml").unlink(missing_ok=True) - (folder / "sleep").unlink(missing_ok=True) if not (folder / "compose.yaml").exists(): return f"{name} is not deployed." - return run( - "docker", "compose", "--project-directory", str(folder), "-p", name, - "--env-file", str(folder / "secrets.env"), "down", "--remove-orphans", - ) + return run("docker", "compose", "--project-directory", str(folder), "-p", name, *compose_options(name), "down", "--remove-orphans") + + +def reapply(name): + if not (BASE / name / "compose.yaml").exists(): + raise LookupError(f"{name} is not deployed") + return run(str(BASE / "bin" / "graphite-up"), name) def secrets(name, request): @@ -103,13 +115,17 @@ def do_GET(self): def do_POST(self): body = self.rfile.read(int(self.headers.get("Content-Length", 0))) - if self.path != "/deploy" or not signed( - self.headers.get("X-Graphite-Timestamp", ""), self.headers.get("X-Graphite-Signature", ""), body - ): + if self.path == "/reapply": + allowed = CONSOLE_TOKEN and hmac.compare_digest(self.headers.get("Authorization", "").encode(), b"Bearer " + CONSOLE_TOKEN) + else: + allowed = self.path == "/deploy" and signed( + self.headers.get("X-Graphite-Timestamp", ""), self.headers.get("X-Graphite-Signature", ""), body + ) + if not allowed: self.reply(401, {"error": "unsigned or expired request"}) return request = json.loads(body) - name, action = request.get("name", ""), request.get("action") + name, action = request.get("name", ""), "reapply" if self.path == "/reapply" else request.get("action") if not NAME.match(name): self.reply(400, {"error": f"{name!r} is not a project name"}) return @@ -120,6 +136,8 @@ def do_POST(self): output = down(name) elif action == "secrets": output = secrets(name, request) + elif action == "reapply": + output = reapply(name) else: self.reply(400, {"error": f"unknown action {action!r}"}) return diff --git a/host/bootstrap.sh b/host/bootstrap.sh index d389129..b38d302 100755 --- a/host/bootstrap.sh +++ b/host/bootstrap.sh @@ -26,8 +26,9 @@ chmod 600 /srv/graphite/edge/secrets.env docker network inspect edge >/dev/null 2>&1 || docker network create edge bind="$(docker network inspect bridge -f '{{(index .IPAM.Config 0).Gateway}}')" -printf 'DEPLOY_TOKEN=%s\nBIND=%s\n' "$DEPLOY_TOKEN" "$bind" > /etc/graphite-agent.env -chmod 600 /etc/graphite-agent.env +install -d -m 700 /etc/graphite +printf 'DEPLOY_TOKEN=%s\nBIND=%s\n' "$DEPLOY_TOKEN" "$bind" > /etc/graphite/agent.env +chmod 600 /etc/graphite/agent.env systemctl daemon-reload systemctl enable --now graphite-agent diff --git a/host/graphite-agent.service b/host/graphite-agent.service index 77e0156..93f53a6 100644 --- a/host/graphite-agent.service +++ b/host/graphite-agent.service @@ -4,8 +4,7 @@ After=docker.service Requires=docker.service [Service] -User=deploy -EnvironmentFile=/etc/graphite-agent.env +EnvironmentFile=/etc/graphite/agent.env ExecStart=/usr/bin/python3 /srv/graphite/bin/agent.py Restart=always RestartSec=5 diff --git a/host/graphite-up b/host/graphite-up index 7fdec28..d8a7311 100755 --- a/host/graphite-up +++ b/host/graphite-up @@ -3,12 +3,15 @@ set -eu name=$1 cd "/srv/graphite/$name" -touch secrets.env -chmod 600 secrets.env -docker compose -p "$name" --env-file secrets.env up -d --wait --wait-timeout 300 --remove-orphans -docker image prune -af --filter until=24h >/dev/null || echo "graphite-up: image prune failed" >&2 - -if [ -f sleep ]; then - curl -fsS -o /dev/null --max-time 90 \ - "http://127.0.0.1:10000/api/strategies/blocking?group=$name&session_duration=$(cat sleep)&timeout=1m" +env=/etc/graphite/$name.env +if [ ! -f "$env" ]; then + env=secrets.env + touch secrets.env + chmod 600 secrets.env fi +set -- +if [ -f values ]; then + set -- --env-file values +fi +docker compose -p "$name" "$@" --env-file "$env" up -d --wait --wait-timeout 300 --remove-orphans +docker image prune -af --filter until=24h >/dev/null || echo "graphite-up: image prune failed" >&2 diff --git a/people/amrmscll.yml b/people/amrmscll.yml deleted file mode 100644 index 148f0f1..0000000 --- a/people/amrmscll.yml +++ /dev/null @@ -1,32 +0,0 @@ -handle: amrmscll -github: AmrMsCLL -profile: manual -name: Amr El-Sheraey -headline: Full-stack Software Engineer -bio: > - Computer Science and AI graduate of FCAI, Cairo University, building document-intelligence - software at Syntheia. A normal week touches a TypeScript API, a React front end, a Python ML service and a Helm chart. Most useful where those meet, and most interested in whichever part I have not done before. -stack: - - group: languages - items: [TypeScript, JavaScript, Python, C++, SQL] - - group: backend - items: [NestJS, Express, FastAPI, Django, Prisma, GraphQL] - - group: frontend - items: [React, Redux Toolkit, Tailwind, MUI, Vite, Office.js] - - group: data - items: [PostgreSQL, MongoDB, Redis, Qdrant, Milvus] - - group: ml - items: [PyTorch, Transformers, spaCy, scikit-learn, Celery] - - group: platform - items: [Docker, Kubernetes, Helm, Terraform, AWS, Azure] - - group: testing - items: [Jest, Pytest, Playwright, SonarQube] -links: - - label: github - url: https://github.com/AmrMsCLL - - label: linkedin - url: https://www.linkedin.com/in/amrsherieycll - - label: email - url: mailto:amrsherieycll@gmail.com - - label: discord - url: https://discord.com/users/412317378337046529 diff --git a/people/bluocaroot.yml b/people/bluocaroot.yml deleted file mode 100644 index a3f0d6c..0000000 --- a/people/bluocaroot.yml +++ /dev/null @@ -1,30 +0,0 @@ -handle: bluocaroot -github: BluoCaroot -profile: manual -name: Assem Amr Abdelfattah -headline: Software Engineer at Syntheia -bio: > - Software engineer from Egypt, building document-intelligence software at Syntheia: a NestJS - API, a React portal and an Outlook add-in on Office.js. Came up through competitive programming, - which still shapes how I approach problems, and outside work I build tools like Digital to - Physical to learn the parts of the stack I have not touched yet. -stack: - - group: languages - items: [TypeScript, JavaScript, C++, Python, C, SQL] - - group: backend - items: [NestJS, Node.js, Express, GraphQL] - - group: frontend - items: [React, MUI, Tailwind, Vite, Office.js] - - group: data - items: [PostgreSQL, MongoDB] - - group: platform - items: [Docker, Azure, Linux, Git] - - group: testing - items: [Jest, SonarQube] -links: - - label: github - url: https://github.com/BluoCaroot - - label: linkedin - url: https://www.linkedin.com/in/assem-amr-abdelfattah/ - - label: codeforces - url: https://codeforces.com/profile/BluoCaroot diff --git a/people/mahmoudhanyfathalla.yml b/people/mahmoudhanyfathalla.yml deleted file mode 100644 index fde1d14..0000000 --- a/people/mahmoudhanyfathalla.yml +++ /dev/null @@ -1,7 +0,0 @@ -handle: mahmoudhanyfathalla -github: MahmoudHanyFathalla -profile: github -name: Mahmoud Hany Fathalla -links: -- label: github - url: https://github.com/MahmoudHanyFathalla diff --git a/people/nier0o.yml b/people/nier0o.yml deleted file mode 100644 index d9ae460..0000000 --- a/people/nier0o.yml +++ /dev/null @@ -1,7 +0,0 @@ -handle: nier0o -github: Nier0o -profile: github -name: Ahmed Saber -links: -- label: github - url: https://github.com/Nier0o diff --git a/people/omar-mohamed24.yml b/people/omar-mohamed24.yml deleted file mode 100644 index 0c0ebb6..0000000 --- a/people/omar-mohamed24.yml +++ /dev/null @@ -1,34 +0,0 @@ -handle: omar-mohamed24 -github: Omar-Mohamed24 -profile: manual -name: Omar Mohamed -headline: Backend & Full-stack Software Engineer -bio: > - Software engineer who goes looking for the hard parts. Wrote a Game Boy emulator from scratch in - C++20, compiled it to WebAssembly and shipped it as PixelPocket, where it passes all of Blargg's - CPU tests and plays in any browser. Built GameForge, a ten-service platform where an LLM planner, - an MCP asset pipeline and a Godot build step turn one sentence into a playable game. Codeforces - Expert with 1000+ problems solved, CS graduate of FCAI, Cairo University, and formerly a prompt - engineer at Scale AI. -stack: - - group: languages - items: [C++, JavaScript, Python, Java, C#] - - group: backend - items: [Node.js, Express, Django, REST APIs, Microservices] - - group: frontend - items: [React, WebAssembly, HTML, CSS] - - group: data - items: [MongoDB, MySQL, MinIO] - - group: ai - items: [LLM Integration, MCP, Stability AI, Prompt Engineering] - - group: platform - items: [Docker, Docker Compose, GitHub Actions, Linux, Git] -links: - - label: github - url: https://github.com/Omar-Mohamed24 - - label: linkedin - url: https://www.linkedin.com/in/omar-mohamed-8034032a3 - - label: email - url: mailto:omarmohamed67371@gmail.com - - label: codeforces - url: https://codeforces.com/profile/Maliiketh diff --git a/schema/person.schema.json b/schema/person.schema.json deleted file mode 100644 index fa0c8a2..0000000 --- a/schema/person.schema.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/project-graphite/platform/blob/main/schema/person.schema.json", - "title": "Project Graphite person entry", - "type": "object", - "additionalProperties": false, - "required": ["handle", "github", "name"], - "properties": { - "handle": { - "type": "string", - "pattern": "^[a-z0-9](?:[a-z0-9-]{0,30}[a-z0-9])?$", - "description": "URL segment for this person. project-graphite.com/members/" - }, - "github": { - "type": "string", - "pattern": "^[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?$" - }, - "profile": { "enum": ["github", "manual"] }, - "name": { "type": "string", "minLength": 2, "maxLength": 60 }, - "headline": { "type": "string", "minLength": 4, "maxLength": 100 }, - "bio": { "type": "string", "minLength": 20, "maxLength": 600 }, - "stack": { - "type": "array", - "maxItems": 8, - "items": { - "type": "object", - "additionalProperties": false, - "required": ["group", "items"], - "properties": { - "group": { "type": "string", "minLength": 2, "maxLength": 20 }, - "items": { - "type": "array", - "minItems": 1, - "maxItems": 12, - "uniqueItems": true, - "items": { "type": "string", "minLength": 1, "maxLength": 24 } - } - } - } - }, - "links": { - "type": "array", - "maxItems": 6, - "uniqueItems": true, - "items": { - "type": "object", - "additionalProperties": false, - "required": ["label", "url"], - "properties": { - "label": { "type": "string", "minLength": 2, "maxLength": 20 }, - "url": { "type": "string", "pattern": "^(https://|mailto:)" } - } - } - } - } -} diff --git a/scripts/check-memory.py b/scripts/check-memory.py new file mode 100644 index 0000000..b652663 --- /dev/null +++ b/scripts/check-memory.py @@ -0,0 +1,48 @@ +import re +import sys + +import yaml + +import deploy + +UNITS = {"": 1, "b": 1, "k": 1024, "m": 1024**2, "g": 1024**3} + + +def mebibytes(value): + amount, unit = re.fullmatch(r"([0-9]+)([bkmg]?)b?", str(value).lower()).groups() + return int(amount) * UNITS[unit] // 1024**2 + + +def main(): + rows = [] + problems = [] + for slug in filter(deploy.compose_project, deploy.projects()): + try: + config = deploy.render(slug)[0] + except ValueError as error: + problems.append(str(error)) + continue + resources = {name: service.get("deploy", {}).get("resources", {}) for name, service in config["services"].items()} + problems.extend(f"deploy/{slug}: {name} has no memory limit" for name, entry in resources.items() if not entry.get("limits", {}).get("memory")) + rows.append(( + slug, + sum(mebibytes(entry.get("limits", {}).get("memory", 0)) for entry in resources.values()), + sum(mebibytes(entry.get("reservations", {}).get("memory", 0)) for entry in resources.values()), + )) + + host = yaml.safe_load(deploy.read(deploy.DEPLOY / "host.yaml")) + budget = mebibytes(host["memory"]) - mebibytes(host["system"]) + reserved = sum(row[2] for row in rows) + print(f"{'project':<24}{'limit':>8}{'reserve':>9}") + for slug, limit, reserve in rows: + print(f"{slug:<24}{limit:>7}M{reserve:>8}M") + print(f"{'total':<24}{sum(row[1] for row in rows):>7}M{reserved:>8}M budget {budget}M") + if reserved > budget: + problems.append(f"reserved memory {reserved}M exceeds the {budget}M budget in deploy/host.yaml") + for problem in problems: + print(f"error: {problem}", file=sys.stderr) + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check-values.py b/scripts/check-values.py new file mode 100644 index 0000000..73dbd34 --- /dev/null +++ b/scripts/check-values.py @@ -0,0 +1,58 @@ +import re +import sys + +import deploy + +REFERENCE = re.compile(r"(?", file=sys.stderr) - return 1 - - projects = [project_card(p) for p in load("projects") if p.get("listed", True)] - projects.sort(key=lambda entry: entry["name"].lower()) - - people = [person_card(p) for p in load("people")] - people.sort(key=lambda entry: entry["name"].lower()) - - pathlib.Path(sys.argv[1]).write_text( - json.dumps({"members": people, "projects": projects}, indent=2) + "\n", encoding="utf-8" - ) - print(f"Compiled {len(people)} members and {len(projects)} projects to {sys.argv[1]}.") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/deploy.py b/scripts/deploy.py index 2c4f6cb..63681fa 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -1,15 +1,10 @@ -import io import json import os import pathlib import re -import shlex -import shutil import subprocess import sys -import tarfile import tempfile -import time import urllib.request import yaml @@ -19,93 +14,93 @@ ROOT = pathlib.Path(__file__).resolve().parent.parent DEPLOY = ROOT / "deploy" -CHART = ROOT / "charts" / "core" -CURL = "curlimages/curl:8.22.0" +EDGE = ROOT / "edge" DEPLOY_HOST = "deploy.project-graphite.com" -COMPONENTS = ("postgres", "redis", "backup") -CHANGE = re.compile(r"^(?:deploy|tags|projects)/([a-z0-9-]+)(?:/|[.]json$|[.]yml$)") -REFERENCE = re.compile(r"(?8}{'reserve':>9} enabled") - for slug, limit, reserve, on in rows: - print(f"{slug:<24}{limit:>7}M{reserve:>8}M {'yes' if on else 'no'}") - print(f"{'total':<24}{sum(row[1] for row in rows):>7}M{reserved:>8}M budget {budget}M") - if reserved > budget: - problems.append(f"reserved memory {reserved}M exceeds the {budget}M budget in deploy/host.yaml") - + problems.extend(f"deploy/{slug}/values: {name} is not used by a Pages site" for name in set(values(slug)) - {"IMAGE_TAG"}) + elif kind == "compose" and compose_project(slug): + route(slug, render(slug)[0]) + else: + problems.append(f"deploy/{slug}: needs compose.yaml and deploy.kind compose in projects/{slug}.yml, or deploy.kind pages") + except ValueError as error: + problems.append(str(error)) for problem in problems: print(f"error: {problem}", file=sys.stderr) + print(f"Checked {len(projects())} projects, found {len(problems)} problems.") return 1 if problems else 0 COMMANDS = { - "release-core": release_core, "plan": show_plan, "apply": apply, "check": check, diff --git a/scripts/people.py b/scripts/people.py deleted file mode 100644 index 600114c..0000000 --- a/scripts/people.py +++ /dev/null @@ -1,91 +0,0 @@ -import json -import sys -import urllib.request - -import yaml - -import sync -import validate - -LABELS = {"generic": "website"} - - -def get(path): - with urllib.request.urlopen(sync.request(path)) as response: - return json.load(response) - - -def members(): - page = 1 - while True: - entries = get(f"/orgs/{sync.ORGANIZATION}/members?per_page=100&page={page}") - yield from entries - if len(entries) < 100: - return - page += 1 - - -def links(user): - found = [{"label": "github", "url": user["html_url"]}] - if user.get("blog"): - blog = user["blog"] if "://" in user["blog"] else f"https://{user['blog']}" - found.append({"label": "website", "url": blog}) - found.extend( - {"label": LABELS.get(account["provider"], account["provider"]), "url": account["url"]} - for account in get(f"/users/{user['login']}/social_accounts") - ) - - unique = [] - for link in found: - if link["url"].startswith("https://") and link not in unique: - unique.append(link) - return unique[:6] - - -def from_github(person, user): - profile = {key: value for key, value in person.items() if key != "headline"} - profile["name"] = user["name"] or user["login"] - headline = " ".join((user["bio"] or "").split()) - if 4 <= len(headline) <= 100: - profile["headline"] = headline - profile["links"] = links(user) - return profile - - -def main(): - people = { - person["github"].lower(): (path, person) - for path, person in validate.load("people") - if isinstance(person, dict) and isinstance(person.get("github"), str) - } - changed = [] - - for member in members(): - login = member["login"] - path, person = people.get( - login.lower(), - ( - sync.ROOT / "people" / f"{login.lower()}.yml", - {"handle": login.lower(), "github": login, "profile": "github"}, - ), - ) - if person.get("profile") == "manual": - continue - - updated = from_github(person, get(f"/users/{login}")) - if updated != person: - path.write_text( - yaml.safe_dump(updated, sort_keys=False, allow_unicode=True, width=100), - encoding="utf-8", - ) - changed.append(updated["handle"]) - - if changed: - print(f"Updated {len(changed)} people from GitHub profiles: {', '.join(sorted(changed))}.") - else: - print("People already match their GitHub profiles.") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/sync.py b/scripts/sync.py index e0bc59f..e36fd20 100644 --- a/scripts/sync.py +++ b/scripts/sync.py @@ -1,5 +1,4 @@ import json -import os import pathlib import re import sys @@ -11,43 +10,18 @@ import validate ROOT = pathlib.Path(__file__).resolve().parent.parent -ORGANIZATION = "Project-Graphite" SLUG = json.loads((ROOT / "schema" / "project.schema.json").read_text(encoding="utf-8"))["properties"]["slug"]["pattern"] -def request(path, accept="application/vnd.github+json"): - token = os.environ.get("GH_TOKEN") - if not token: - raise RuntimeError("GH_TOKEN is required") - return urllib.request.Request( - f"https://api.github.com{path}", - headers={ - "Accept": accept, - "Authorization": f"Bearer {token}", - "X-GitHub-Api-Version": "2022-11-28", - "User-Agent": "project-graphite-registry-sync", - }, - ) - - def repositories(): - page = 1 - while True: - with urllib.request.urlopen( - request(f"/orgs/{ORGANIZATION}/repos?type=all&per_page=100&page={page}") - ) as response: - entries = json.load(response) - yield from entries - if len(entries) < 100: - return - page += 1 + return validate.paginate(f"/orgs/{validate.ORGANIZATION}/repos?type=all") def manifest(repository): try: with urllib.request.urlopen( - request( - f"/repos/{ORGANIZATION}/{repository}/contents/.graphite.yml", + validate.request( + f"/repos/{validate.ORGANIZATION}/{repository}/contents/.graphite.yml", "application/vnd.github.raw+json", ) ) as response: @@ -66,7 +40,7 @@ def entries(projects): return [(ROOT / "projects" / f"{slug}.yml", project) for slug, project in sorted(projects.items())] -def apply(name, text, projects, organisation, people): +def apply(name, text, projects, organisation, members): try: source = yaml.safe_load(text) except yaml.YAMLError as error: @@ -103,9 +77,9 @@ def apply(name, text, projects, organisation, people): for slug, project in projects.items() if not (isinstance(project, dict) and project.get("repo") == repo) } | candidates - existing = set(validate.registry_problems(entries(projects), people)) + existing = set(validate.registry_problems(entries(projects), members)) problems.extend( - line for line in validate.registry_problems(entries(updated), people) if line not in existing + line for line in validate.registry_problems(entries(updated), members) if line not in existing ) return updated, problems @@ -115,25 +89,10 @@ def report(name, problems): print(f"error: {name}: {line}", file=sys.stderr) -def check(name, path): - _, problems = apply( - name, - pathlib.Path(path).read_text(encoding="utf-8"), - registry(), - {repository["name"].lower() for repository in repositories()}, - validate.load("people"), - ) - report(name, problems) - if problems: - return 1 - print(f"{path} is a valid manifest for {name}.") - return 0 - - def sync(): existing = registry() projects = existing - people = validate.load("people") + members = validate.members() listed = list(repositories()) organisation = {repository["name"].lower() for repository in listed} skipped = [] @@ -142,7 +101,7 @@ def sync(): text = manifest(repository["name"]) if text is None: continue - updated, problems = apply(repository["name"], text, projects, organisation, people) + updated, problems = apply(repository["name"], text, projects, organisation, members) if problems: report(repository["name"], problems) skipped.append(repository["name"]) @@ -174,14 +133,5 @@ def sync(): return 0 -def main(): - if len(sys.argv) == 1: - return sync() - if len(sys.argv) == 3: - return check(*sys.argv[1:]) - print("usage: sync.py [ ]", file=sys.stderr) - return 1 - - if __name__ == "__main__": - sys.exit(main()) + sys.exit(sync()) diff --git a/scripts/validate.py b/scripts/validate.py index e42c5ff..11ceaba 100644 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -1,19 +1,45 @@ import json +import os import pathlib import sys +import urllib.request import jsonschema import yaml ROOT = pathlib.Path(__file__).resolve().parent.parent +ORGANIZATION = "Project-Graphite" +PROJECT = jsonschema.Draft202012Validator(json.loads((ROOT / "schema" / "project.schema.json").read_text(encoding="utf-8"))) + + +def request(path, accept="application/vnd.github+json"): + token = os.environ.get("GH_TOKEN") + if not token: + raise RuntimeError("GH_TOKEN is required") + return urllib.request.Request( + f"https://api.github.com{path}", + headers={ + "Accept": accept, + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "project-graphite-registry-sync", + }, + ) -def validator_for(name): - return jsonschema.Draft202012Validator(json.loads((ROOT / "schema" / name).read_text(encoding="utf-8"))) +def paginate(path): + page = 1 + while True: + with urllib.request.urlopen(request(f"{path}{'&' if '?' in path else '?'}per_page=100&page={page}")) as response: + entries = json.load(response) + yield from entries + if len(entries) < 100: + return + page += 1 -PROJECT = validator_for("project.schema.json") -PERSON = validator_for("person.schema.json") +def members(): + return {member["login"].lower() for member in paginate(f"/orgs/{ORGANIZATION}/members")} def load(directory): @@ -63,16 +89,11 @@ def uniqueness_problems(entries, value_of, label): claimed[key] = path.name -def ownership_problems(projects, people): - handles = { - person["handle"] - for _, person in people - if isinstance(person, dict) and isinstance(person.get("handle"), str) - } +def ownership_problems(projects, organisation): for path, project in projects: owner = project.get("owner") if isinstance(project, dict) else None - if owner and isinstance(owner, str) and owner not in handles: - yield f"{path.name}: owner {owner} has no entry in people/" + if owner and isinstance(owner, str) and owner.lower() not in organisation: + yield f"{path.name}: owner {owner} is not a member of {ORGANIZATION}" def hierarchy_problems(projects): @@ -102,17 +123,9 @@ def hierarchy_problems(projects): yield f"{path.name}: child project must use parent repository {parent.get('repo')}" -def registry_problems(projects, people): - problems = [line for path, person in people for line in schema_problems(PERSON, path, person, "handle")] - problems.extend(line for path, project in projects for line in project_problems(path, project)) - problems.extend( - uniqueness_problems( - people, - lambda person: person.get("github") if isinstance(person, dict) else None, - "github account", - ) - ) - problems.extend(ownership_problems(projects, people)) +def registry_problems(projects, organisation): + problems = [line for path, project in projects for line in project_problems(path, project)] + problems.extend(ownership_problems(projects, organisation)) problems.extend(hierarchy_problems(projects)) problems.extend( uniqueness_problems( @@ -126,20 +139,15 @@ def registry_problems(projects, people): def main(): projects = load("projects") - people = load("people") - if not projects: print("No registry entries in projects/.", file=sys.stderr) return 1 - if not people: - print("No entries in people/.", file=sys.stderr) - return 1 - problems = registry_problems(projects, people) + problems = registry_problems(projects, members()) for line in problems: print(f"error: {line}", file=sys.stderr) - print(f"Checked {len(people)} people and {len(projects)} projects, found {len(problems)} problems.") + print(f"Checked {len(projects)} projects, found {len(problems)} problems.") return 1 if problems else 0 diff --git a/shared/app.yaml b/shared/app.yaml new file mode 100644 index 0000000..d5bf087 --- /dev/null +++ b/shared/app.yaml @@ -0,0 +1,32 @@ +services: + service: + init: true + restart: unless-stopped + logging: + driver: json-file + options: {max-size: 10m, max-file: "5"} + security_opt: + - no-new-privileges:true + healthcheck: + interval: 30s + timeout: 5s + retries: 3 + start_period: 0s + deploy: + resources: + limits: {cpus: "0.5", memory: 256M} + reservations: {memory: 64M} + + app: + extends: service + expose: ["${APP_PORT:-3000}"] + environment: + APP_URL: https://${HOST} + networks: + default: {} + edge: + aliases: ["${PROJECT}"] + deploy: + resources: + limits: {cpus: "${APP_CPUS:-0.5}", memory: "${APP_MEMORY:-256M}"} + reservations: {memory: "${APP_RESERVE:-64M}"} diff --git a/shared/postgres.yaml b/shared/postgres.yaml new file mode 100644 index 0000000..06af269 --- /dev/null +++ b/shared/postgres.yaml @@ -0,0 +1,20 @@ +services: + postgres: + extends: {file: app.yaml, service: service} + image: postgres:18-alpine + command: [postgres, -c, "shared_buffers=${POSTGRES_SHARED_BUFFERS:-128MB}", -c, "max_connections=${POSTGRES_MAX_CONNECTIONS:-100}"] + environment: + POSTGRES_DB: ${PROJECT} + POSTGRES_USER: ${PROJECT} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?} + volumes: [postgres-data:/var/lib/postgresql] + labels: {graphite.backup.postgres: "true"} + healthcheck: + test: [CMD-SHELL, "pg_isready -U \"$$POSTGRES_USER\" -d \"$$POSTGRES_DB\""] + interval: 10s + retries: 10 + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "${POSTGRES_CPUS:-0.5}", memory: "${POSTGRES_MEMORY:-256M}"} + reservations: {memory: "${POSTGRES_RESERVE:-64M}"} diff --git a/shared/redis-password.yaml b/shared/redis-password.yaml new file mode 100644 index 0000000..bc95abf --- /dev/null +++ b/shared/redis-password.yaml @@ -0,0 +1,8 @@ +services: + redis: + extends: {file: redis.yaml, service: redis} + command: [sh, -c, "exec redis-server --appendonly yes --maxmemory ${REDIS_MAXMEMORY:-48mb} --maxmemory-policy ${REDIS_POLICY:-allkeys-lru} --requirepass \"$$REDIS_PASSWORD\""] + environment: + REDIS_PASSWORD: ${REDIS_PASSWORD:?} + healthcheck: + test: [CMD-SHELL, "REDISCLI_AUTH=\"$$REDIS_PASSWORD\" redis-cli ping | grep -q PONG"] diff --git a/shared/redis.yaml b/shared/redis.yaml new file mode 100644 index 0000000..3757a09 --- /dev/null +++ b/shared/redis.yaml @@ -0,0 +1,14 @@ +services: + redis: + extends: {file: app.yaml, service: service} + image: redis:8.10-alpine + command: [redis-server, --appendonly, "yes", --maxmemory, "${REDIS_MAXMEMORY:-48mb}", --maxmemory-policy, "${REDIS_POLICY:-allkeys-lru}"] + volumes: [redis-data:/data] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 10s + retries: 10 + deploy: + resources: + limits: {cpus: "0.25", memory: "${REDIS_MEMORY:-64M}"} + reservations: {memory: "${REDIS_RESERVE:-16M}"} diff --git a/tags/console.json b/tags/console.json deleted file mode 100644 index 165b401..0000000 --- a/tags/console.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "app": "sha-8d9bb17" -} diff --git a/tags/digital-to-physical.json b/tags/digital-to-physical.json deleted file mode 100644 index cd6d273..0000000 --- a/tags/digital-to-physical.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "site": "sha-d5ba7b2" -} diff --git a/tags/graphite-hub.json b/tags/graphite-hub.json deleted file mode 100644 index 185df70..0000000 --- a/tags/graphite-hub.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "site": "sha-ff5f3e8" -} diff --git a/tags/graphite-message-lab.json b/tags/graphite-message-lab.json deleted file mode 100644 index 06f012e..0000000 --- a/tags/graphite-message-lab.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "web": "sha-599fbe4", - "worker": "sha-599fbe4" -} diff --git a/tags/graphite-tracker.json b/tags/graphite-tracker.json deleted file mode 100644 index 483ab49..0000000 --- a/tags/graphite-tracker.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "app": "sha-7533f37" -} diff --git a/tags/pixelpocket.json b/tags/pixelpocket.json deleted file mode 100644 index 67a977c..0000000 --- a/tags/pixelpocket.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "backend": "sha-e38c8e1", - "frontend": "sha-e38c8e1" -} diff --git a/tags/strata.json b/tags/strata.json deleted file mode 100644 index 5f25170..0000000 --- a/tags/strata.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "app": "sha-ebcde55" -}