fix: BED-10186 publish latest images and Compose setup

.github/workflows/publish-container.yml

@@ -56,6 +56,7 @@ jobs:
           if [[ "$TARGET" == cli ]]; then SUFFIX="-cli"; fi
           echo "version=$VERSION" >> "$GITHUB_OUTPUT"
           echo "tag=$IMAGE:$VERSION$SUFFIX" >> "$GITHUB_OUTPUT"
+          echo "latest_tag=$IMAGE:latest$SUFFIX" >> "$GITHUB_OUTPUT"
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v4
@@ -125,3 +126,30 @@ jobs:
           build-args: |
             COLLECTOR_VERSION=${{ steps.image.outputs.version }}
           cache-from: type=gha,scope=${{ matrix.target }}
+
+      - name: Update latest alias for newest release
+        env:
+          RELEASE_TAG: ${{ inputs.tag }}
+          VERSIONED_TAG: ${{ steps.image.outputs.tag }}
+          LATEST_TAG: ${{ steps.image.outputs.latest_tag }}
+        run: |
+          NEWEST_RELEASE_TAG=$(
+            git ls-remote --refs --tags origin 'refs/tags/v*' |
+              awk '$2 ~ /^refs\/tags\/v[0-9]+\.[0-9]+\.[0-9]+$/ {
+                sub("^refs/tags/", "", $2)
+                print $2
+              }' |
+              sort -V |
+              tail -n 1
+          )
+          if [[ -z "$NEWEST_RELEASE_TAG" ]]; then
+            echo "Unable to determine the newest release tag" >&2
+            exit 1
+          fi
+          if [[ "$RELEASE_TAG" != "$NEWEST_RELEASE_TAG" ]]; then
+            echo "Skipping $LATEST_TAG: newest release is $NEWEST_RELEASE_TAG"
+            exit 0
+          fi
+          docker buildx imagetools create \
+            --tag "$LATEST_TAG" \
+            "$VERSIONED_TAG"

.gitignore

@@ -22,6 +22,11 @@ __pycache__/
 *.py[codz]
 *$py.class
 .dlt
+# Publish only the Compose configuration examples, never local credentials.
+!/docker-compose/.dlt/
+/docker-compose/.dlt/*
+!/docker-compose/.dlt/config.toml.example
+!/docker-compose/.dlt/secrets.toml.example
 # C extensions
 *.so
 

README.md

@@ -157,10 +157,18 @@ tag pushes. Both images use the same release version:
 
 For example, release tag `v0.20.0` produces image tags `0.20.0` and `0.20.0-cli`.
 Use a version containing the container files; releases predating this workflow
-cannot be built without backporting those files. No moving `latest` tag is published.
+cannot be built without backporting those files.
+
+Each release also updates `latest` for the scheduler and `latest-cli` for the
+CLI. These aliases point to the same images as that release's versioned tags.
+Use versioned tags to pin a deployment. To update a Compose deployment using
+`latest`, run `docker compose pull` followed by `docker compose up -d`.
 
 ### Run the scheduler
 
+For a complete Compose setup with configuration templates, persistent storage,
+and a preflight check, see [docker-compose/README.md](docker-compose/README.md).
+
 Prepare your `.dlt/config.toml`, `.dlt/secrets.toml`, and GitHub App private key
 following the collector configuration documentation above. Mount the directory
 read-only and use the container path for the key (for example,

docker-compose/.dlt/config.toml.example

@@ -0,0 +1,5 @@
+[runtime]
+log_cli_level = "INFO"
+
+[destination.bloodhoundenterprise]
+collector_name = "github"

docker-compose/.dlt/secrets.toml.example

@@ -0,0 +1,12 @@
+[destination.bloodhoundenterprise]
+url = "https://REPLACE_WITH_TENANT.bloodhoundenterprise.io"
+# Credentials for the GitHub OpenHound collector client in BHE.
+token_id = "REPLACE_WITH_BHE_COLLECTOR_TOKEN_ID"
+token_key = "REPLACE_WITH_BHE_COLLECTOR_TOKEN_KEY"
+
+[sources.source.github.credentials]
+client_id = "REPLACE_WITH_GITHUB_APP_CLIENT_ID"
+enterprise_name = "REPLACE_WITH_ENTERPRISE_SLUG"
+key_path = "/app/.dlt/github-app.pem"
+# Optional classic PAT for enterprise SAML SSO and SCIM collection:
+# pat_token = "REPLACE_WITH_OPTIONAL_PAT"

docker-compose/.env.example

@@ -0,0 +1,2 @@
+# Follow published scheduler releases. The -cli image does not poll BHE for jobs.
+OPENHOUND_IMAGE=ghcr.io/specterops/openhound-github:latest

docker-compose/README.md

@@ -0,0 +1,150 @@
+# Run the GitHub collector with Docker Compose
+
+Run the published GitHub scheduler image against a BloodHound Enterprise (BHE)
+tenant using an enterprise GitHub App. The images support Linux AMD64 and
+ARM64, and Docker automatically selects the image for the host architecture.
+
+The scheduler polls BHE for jobs. Starting the container does not start a
+collection: run an on-demand scan or configure a schedule in BHE.
+
+## Prerequisites
+
+- Docker Engine with the Compose plugin on Linux, or Docker Desktop on macOS.
+- A BHE tenant with OpenHound enabled and the GitHub extension assets installed.
+- A GitHub OpenHound collector client in BHE and its token ID and token key.
+  These are collector-client credentials, not a general user's API token.
+- An enterprise GitHub App installed in the enterprise and the organizations
+  you intend to collect, its Client ID, and its private key.
+
+See [BHE deployment instructions](https://bloodhound.specterops.io/openhound/deploy-with-compose)
+and [enterprise GitHub App configuration](https://bloodhound.specterops.io/openhound/collectors/github/configure-enterprise-app)
+for collector registration and GitHub permissions. Install the SCIM extension
+if collecting SCIM data. An optional classic PAT with `read:enterprise` enables
+enterprise SAML SSO and SCIM collection; `manage_runners:enterprise` is needed
+only if you also want enterprise self-hosted runner data.
+
+## Prepare configuration
+
+From the repository root:
+
+```sh
+cd docker-compose
+cp .env.example .env
+cp .dlt/config.toml.example .dlt/config.toml
+cp .dlt/secrets.toml.example .dlt/secrets.toml
+cp /path/to/downloaded-key.pem .dlt/github-app.pem
+```
+
+The resulting layout is:
+
+```text
+docker-compose/
+├── compose.yaml
+├── check_config.py
+├── .env
+└── .dlt/
+    ├── config.toml
+    ├── secrets.toml
+    └── github-app.pem
+```
+
+Edit `.dlt/secrets.toml` locally and replace the placeholders:
+
+| Setting | Value |
+| --- | --- |
+| BHE `url` | Your BHE tenant's HTTPS URL |
+| BHE `token_id`, `token_key` | The GitHub OpenHound collector client's credentials |
+| GitHub `client_id` | The enterprise GitHub App's Client ID |
+| GitHub `enterprise_name` | The slug from `github.com/enterprises/<slug>` |
+| GitHub `key_path` | Leave `/app/.dlt/github-app.pem` when using the layout above |
+
+Actual `.env`, `.dlt` configuration files, and private keys are ignored by Git.
+Only the example files are intended for publication. The `.dlt` directory is
+mounted read-only into the container.
+
+The container runs as UID/GID `65532` on every host. Its mounted files must be
+readable by that user. On Linux and macOS, keep this host directory owner-only
+while allowing the container to read the mounted copies:
+
+```sh
+chmod 700 .
+chmod 755 .dlt
+chmod 644 .dlt/config.toml .dlt/secrets.toml .dlt/github-app.pem
+```
+
+These commands affect the local key copy, not the original downloaded key. The
+owner-only parent directory protects the host copies while the mounted `.dlt`
+directory and files remain readable inside the container. On Linux systems
+with enforcing SELinux, the bind-mounted `.dlt` directory must also have a
+container-readable SELinux label permitted by the host policy.
+
+## Validate and start
+
+```sh
+docker compose config --quiet
+docker compose pull
+docker compose run --rm --entrypoint python scheduler-github /opt/openhound-check-config.py
+docker compose up -d scheduler-github
+docker compose logs -f scheduler-github
+```
+
+The preflight checks configuration resolution, RSA private-key readability,
+and writable storage. It does not authenticate to GitHub or BHE. The one-shot
+`storage-init` service prepares the named volumes before the scheduler starts;
+the scheduler itself runs as non-root.
+
+In BHE, run an on-demand scan for the matching collector client. Follow the
+container logs and BHE's collection, ingest, and analysis status. API rate-limit
+retries can make a large collection take longer.
+
+## Image versions and updates
+
+`.env.example` follows the `latest` scheduler release. Set `OPENHOUND_IMAGE` in
+`.env` to a specific published version if you need a fixed deployment. Use the
+scheduler image, without the `-cli` suffix.
+
+After changing the image, or to update a deployment using `latest`, run:
+
+```sh
+docker compose pull
+docker compose up -d scheduler-github
+```
+
+A running container does not update automatically when a tag changes in GHCR.
+
+## Storage and operations
+
+| Named volume | Contents |
+| --- | --- |
+| `collector-data` | DLT state, the DuckDB lookup database, and application logs under `/var/lib/openhound` |
+| `raw-data` | Raw collection output under `/tmp/openhound` |
+
+```sh
+docker compose ps -a
+docker compose logs --tail 100 scheduler-github
+docker compose stop scheduler-github
+docker compose down
+```
+
+Normal `down` and container recreation preserve the named volumes.
+`docker compose down --volumes` deletes them and their stored collection data.
+Stop the scheduler after a collection completes. Idle shutdown can take up to
+the 30-second polling interval; Compose allows 45 seconds before forcing exit.
+
+The collector needs outbound access to GitHub and BHE; no inbound host ports
+are required. Each deployment should have its own BHE collector client and
+Compose project name. Use `docker compose -p <name> ...` consistently if running
+multiple copies so their containers and volumes remain separate.
+
+## Troubleshooting
+
+- **Preflight reports a placeholder:** fill in `.dlt/secrets.toml` and rerun it.
+- **Private key or secrets cannot be read:** verify the mounted file paths and
+  UID `65532` read permissions. On Linux with enforcing SELinux, also verify
+  that policy permits the `.dlt` bind mount. The container key path is
+  different from the path of the downloaded key on your host.
+- **BHE authentication fails:** check the tenant URL and the matching GitHub
+  OpenHound collector client's token ID/key.
+- **Scheduler is idle:** start an on-demand scan or schedule a collection in BHE.
+- **GitHub authentication fails:** confirm the key belongs to the configured
+  app and that the app is installed in the enterprise and target organizations.

docker-compose/check_config.py

@@ -0,0 +1,54 @@
+"""Validate local configuration without contacting GitHub or BHE."""
+
+import importlib.metadata
+import os
+import tempfile
+from pathlib import Path
+from urllib.parse import urlparse
+
+import dlt
+from dlt.common.configuration import resolve_configuration
+from joserfc.jwk import RSAKey
+from openhound_github.source import GithubEnterpriseAppCredentials
+
+
+def required(name):
+    value = dlt.secrets.get(name)
+    if value is None or not str(value).strip() or "REPLACE_WITH" in str(value):
+        raise ValueError(f"Set {name} in .dlt/secrets.toml")
+    return str(value)
+
+
+url = required("destination.bloodhoundenterprise.url")
+if urlparse(url).scheme != "https" or not urlparse(url).hostname:
+    raise ValueError("Set a valid HTTPS BHE tenant URL")
+required("destination.bloodhoundenterprise.token_id")
+required("destination.bloodhoundenterprise.token_key")
+required("sources.source.github.credentials.enterprise_name")
+
+credentials = resolve_configuration(
+    GithubEnterpriseAppCredentials(),
+    sections=("sources", "source", "github", "credentials"),
+)
+issuer = credentials.client_id or credentials.app_id
+if not issuer or "REPLACE_WITH" in str(issuer):
+    raise ValueError("Set the GitHub App client_id or app_id")
+key_path = Path(credentials.key_path)
+key = RSAKey.import_key(key_path.read_bytes())
+if not key.is_private:
+    raise ValueError("The GitHub App PEM must contain a private key")
+
+for directory in (
+    Path.cwd(),
+    Path(os.environ["DLT_DATA_DIR"]),
+    Path(os.environ["PIPELINES_DIR"]),
+    Path(os.environ["RUNTIME__LOG_PATH"]),
+    Path("/tmp/openhound"),
+):
+    with tempfile.TemporaryFile(mode="w+", dir=directory) as probe:
+        probe.write("ok")
+
+print("PASS: BHE and enterprise-app configuration resolves; RSA private key is readable.")
+print("PASS: collection, state, lookup, and log directories are writable.")
+print("GitHub collector version:", importlib.metadata.version("openhound-github"))
+print("Preflight does not verify credentials against GitHub or BHE.")

docker-compose/compose.yaml

@@ -0,0 +1,62 @@
+name: openhound-github
+
+services:
+  storage-init:
+    image: ${OPENHOUND_IMAGE:-ghcr.io/specterops/openhound-github:latest}
+    user: "0:0"
+    network_mode: none
+    restart: "no"
+    entrypoint: ["python", "-c"]
+    command:
+      - |
+        import os
+        from pathlib import Path
+        for name in ("/var/lib/openhound", "/var/lib/openhound/work", "/var/lib/openhound/dlt", "/var/lib/openhound/dlt/pipelines", "/var/lib/openhound/logs", "/tmp/openhound"):
+            path = Path(name)
+            path.mkdir(parents=True, exist_ok=True)
+            os.chown(path, 65532, 65532)
+        print("Collector storage is ready.")
+    volumes:
+      - collector-data:/var/lib/openhound
+      - raw-data:/tmp/openhound
+
+  scheduler-github:
+    image: ${OPENHOUND_IMAGE:-ghcr.io/specterops/openhound-github:latest}
+    init: true
+    restart: unless-stopped
+    stop_grace_period: 45s
+    depends_on:
+      storage-init:
+        condition: service_completed_successfully
+    entrypoint: ["python", "/app/src/scheduler.py"]
+    working_dir: /var/lib/openhound/work
+    environment:
+      DLT_PROJECT_DIR: /app
+      DLT_DATA_DIR: /var/lib/openhound/dlt
+      PIPELINES_DIR: /var/lib/openhound/dlt/pipelines
+      RUNTIME__LOG_PATH: /var/lib/openhound/logs
+      DESTINATION__BLOODHOUNDENTERPRISE__COLLECTOR_NAME: github
+    volumes:
+      - type: bind
+        source: ./.dlt
+        target: /app/.dlt
+        read_only: true
+        bind:
+          create_host_path: false
+      - type: bind
+        source: ./check_config.py
+        target: /opt/openhound-check-config.py
+        read_only: true
+        bind:
+          create_host_path: false
+      - collector-data:/var/lib/openhound
+      - raw-data:/tmp/openhound
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
+
+volumes:
+  collector-data:
+  raw-data: