From f5e7379206ca671f2226e4be9e6bdb66f4589ea5 Mon Sep 17 00:00:00 2001 From: Jared Atkinson Date: Fri, 23 Oct 2026 13:39:44 -0700 Subject: [PATCH 1/2] fix: BED-10186 publish latest images and Compose setup --- .github/workflows/publish-container.yml | 5 +- .gitignore | 5 + README.md | 10 +- docker-compose/.dlt/config.toml.example | 5 + docker-compose/.dlt/secrets.toml.example | 12 ++ docker-compose/.env.example | 2 + docker-compose/README.md | 150 +++++++++++++++++++++++ docker-compose/check_config.py | 54 ++++++++ docker-compose/compose.yaml | 62 ++++++++++ 9 files changed, 303 insertions(+), 2 deletions(-) create mode 100644 docker-compose/.dlt/config.toml.example create mode 100644 docker-compose/.dlt/secrets.toml.example create mode 100644 docker-compose/.env.example create mode 100644 docker-compose/README.md create mode 100644 docker-compose/check_config.py create mode 100644 docker-compose/compose.yaml diff --git a/.github/workflows/publish-container.yml b/.github/workflows/publish-container.yml index a310fee..074f498 100644 --- a/.github/workflows/publish-container.yml +++ b/.github/workflows/publish-container.yml @@ -56,6 +56,7 @@ jobs: if [[ "$TARGET" == cli ]]; then SUFFIX="-cli"; fi echo "version=$VERSION" >> "$GITHUB_OUTPUT" echo "tag=$IMAGE:$VERSION$SUFFIX" >> "$GITHUB_OUTPUT" + echo "latest_tag=$IMAGE:latest$SUFFIX" >> "$GITHUB_OUTPUT" - name: Set up QEMU uses: docker/setup-qemu-action@v4 @@ -118,7 +119,9 @@ jobs: platforms: linux/amd64,linux/arm64 push: true sbom: true - tags: ${{ steps.image.outputs.tag }} + tags: | + ${{ steps.image.outputs.tag }} + ${{ steps.image.outputs.latest_tag }} labels: | org.opencontainers.image.source=https://github.com/${{ github.repository }} org.opencontainers.image.version=${{ steps.image.outputs.version }} diff --git a/.gitignore b/.gitignore index cac8bd5..6d66c41 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,11 @@ __pycache__/ *.py[codz] *$py.class .dlt +# Publish only the Compose configuration examples, never local credentials. +!/docker-compose/.dlt/ +/docker-compose/.dlt/* +!/docker-compose/.dlt/config.toml.example +!/docker-compose/.dlt/secrets.toml.example # C extensions *.so diff --git a/README.md b/README.md index 61eb87b..db260df 100644 --- a/README.md +++ b/README.md @@ -157,10 +157,18 @@ tag pushes. Both images use the same release version: For example, release tag `v0.20.0` produces image tags `0.20.0` and `0.20.0-cli`. Use a version containing the container files; releases predating this workflow -cannot be built without backporting those files. No moving `latest` tag is published. +cannot be built without backporting those files. + +Each release also updates `latest` for the scheduler and `latest-cli` for the +CLI. These aliases point to the same images as that release's versioned tags. +Use versioned tags to pin a deployment. To update a Compose deployment using +`latest`, run `docker compose pull` followed by `docker compose up -d`. ### Run the scheduler +For a complete Compose setup with configuration templates, persistent storage, +and a preflight check, see [docker-compose/README.md](docker-compose/README.md). + Prepare your `.dlt/config.toml`, `.dlt/secrets.toml`, and GitHub App private key following the collector configuration documentation above. Mount the directory read-only and use the container path for the key (for example, diff --git a/docker-compose/.dlt/config.toml.example b/docker-compose/.dlt/config.toml.example new file mode 100644 index 0000000..1804e98 --- /dev/null +++ b/docker-compose/.dlt/config.toml.example @@ -0,0 +1,5 @@ +[runtime] +log_cli_level = "INFO" + +[destination.bloodhoundenterprise] +collector_name = "github" diff --git a/docker-compose/.dlt/secrets.toml.example b/docker-compose/.dlt/secrets.toml.example new file mode 100644 index 0000000..73618b0 --- /dev/null +++ b/docker-compose/.dlt/secrets.toml.example @@ -0,0 +1,12 @@ +[destination.bloodhoundenterprise] +url = "https://REPLACE_WITH_TENANT.bloodhoundenterprise.io" +# Credentials for the GitHub OpenHound collector client in BHE. +token_id = "REPLACE_WITH_BHE_COLLECTOR_TOKEN_ID" +token_key = "REPLACE_WITH_BHE_COLLECTOR_TOKEN_KEY" + +[sources.source.github.credentials] +client_id = "REPLACE_WITH_GITHUB_APP_CLIENT_ID" +enterprise_name = "REPLACE_WITH_ENTERPRISE_SLUG" +key_path = "/app/.dlt/github-app.pem" +# Optional classic PAT for enterprise SAML SSO and SCIM collection: +# pat_token = "REPLACE_WITH_OPTIONAL_PAT" diff --git a/docker-compose/.env.example b/docker-compose/.env.example new file mode 100644 index 0000000..88b2db5 --- /dev/null +++ b/docker-compose/.env.example @@ -0,0 +1,2 @@ +# Follow published scheduler releases. The -cli image does not poll BHE for jobs. +OPENHOUND_IMAGE=ghcr.io/specterops/openhound-github:latest diff --git a/docker-compose/README.md b/docker-compose/README.md new file mode 100644 index 0000000..ca261db --- /dev/null +++ b/docker-compose/README.md @@ -0,0 +1,150 @@ +# Run the GitHub collector with Docker Compose + +Run the published GitHub scheduler image against a BloodHound Enterprise (BHE) +tenant using an enterprise GitHub App. The images support Linux AMD64 and +ARM64, and Docker automatically selects the image for the host architecture. + +The scheduler polls BHE for jobs. Starting the container does not start a +collection: run an on-demand scan or configure a schedule in BHE. + +## Prerequisites + +- Docker Engine with the Compose plugin on Linux, or Docker Desktop on macOS. +- A BHE tenant with OpenHound enabled and the GitHub extension assets installed. +- A GitHub OpenHound collector client in BHE and its token ID and token key. + These are collector-client credentials, not a general user's API token. +- An enterprise GitHub App installed in the enterprise and the organizations + you intend to collect, its Client ID, and its private key. + +See [BHE deployment instructions](https://bloodhound.specterops.io/openhound/deploy-with-compose) +and [enterprise GitHub App configuration](https://bloodhound.specterops.io/openhound/collectors/github/configure-enterprise-app) +for collector registration and GitHub permissions. Install the SCIM extension +if collecting SCIM data. An optional classic PAT with `read:enterprise` enables +enterprise SAML SSO and SCIM collection; `manage_runners:enterprise` is needed +only if you also want enterprise self-hosted runner data. + +## Prepare configuration + +From the repository root: + +```sh +cd docker-compose +cp .env.example .env +cp .dlt/config.toml.example .dlt/config.toml +cp .dlt/secrets.toml.example .dlt/secrets.toml +cp /path/to/downloaded-key.pem .dlt/github-app.pem +``` + +The resulting layout is: + +```text +docker-compose/ +├── compose.yaml +├── check_config.py +├── .env +└── .dlt/ + ├── config.toml + ├── secrets.toml + └── github-app.pem +``` + +Edit `.dlt/secrets.toml` locally and replace the placeholders: + +| Setting | Value | +| --- | --- | +| BHE `url` | Your BHE tenant's HTTPS URL | +| BHE `token_id`, `token_key` | The GitHub OpenHound collector client's credentials | +| GitHub `client_id` | The enterprise GitHub App's Client ID | +| GitHub `enterprise_name` | The slug from `github.com/enterprises/` | +| GitHub `key_path` | Leave `/app/.dlt/github-app.pem` when using the layout above | + +Actual `.env`, `.dlt` configuration files, and private keys are ignored by Git. +Only the example files are intended for publication. The `.dlt` directory is +mounted read-only into the container. + +The container runs as UID/GID `65532` on every host. Its mounted files must be +readable by that user. On Linux and macOS, keep this host directory owner-only +while allowing the container to read the mounted copies: + +```sh +chmod 700 . +chmod 755 .dlt +chmod 644 .dlt/config.toml .dlt/secrets.toml .dlt/github-app.pem +``` + +These commands affect the local key copy, not the original downloaded key. The +owner-only parent directory protects the host copies while the mounted `.dlt` +directory and files remain readable inside the container. On Linux systems +with enforcing SELinux, the bind-mounted `.dlt` directory must also have a +container-readable SELinux label permitted by the host policy. + +## Validate and start + +```sh +docker compose config --quiet +docker compose pull +docker compose run --rm --entrypoint python scheduler-github /opt/openhound-check-config.py +docker compose up -d scheduler-github +docker compose logs -f scheduler-github +``` + +The preflight checks configuration resolution, RSA private-key readability, +and writable storage. It does not authenticate to GitHub or BHE. The one-shot +`storage-init` service prepares the named volumes before the scheduler starts; +the scheduler itself runs as non-root. + +In BHE, run an on-demand scan for the matching collector client. Follow the +container logs and BHE's collection, ingest, and analysis status. API rate-limit +retries can make a large collection take longer. + +## Image versions and updates + +`.env.example` follows the `latest` scheduler release. Set `OPENHOUND_IMAGE` in +`.env` to a specific published version if you need a fixed deployment. Use the +scheduler image, without the `-cli` suffix. + +After changing the image, or to update a deployment using `latest`, run: + +```sh +docker compose pull +docker compose up -d scheduler-github +``` + +A running container does not update automatically when a tag changes in GHCR. + +## Storage and operations + +| Named volume | Contents | +| --- | --- | +| `collector-data` | DLT state, the DuckDB lookup database, and application logs under `/var/lib/openhound` | +| `raw-data` | Raw collection output under `/tmp/openhound` | + +```sh +docker compose ps -a +docker compose logs --tail 100 scheduler-github +docker compose stop scheduler-github +docker compose down +``` + +Normal `down` and container recreation preserve the named volumes. +`docker compose down --volumes` deletes them and their stored collection data. +Stop the scheduler after a collection completes. Idle shutdown can take up to +the 30-second polling interval; Compose allows 45 seconds before forcing exit. + +The collector needs outbound access to GitHub and BHE; no inbound host ports +are required. Each deployment should have its own BHE collector client and +Compose project name. Use `docker compose -p ...` consistently if running +multiple copies so their containers and volumes remain separate. + +## Troubleshooting + +- **Preflight reports a placeholder:** fill in `.dlt/secrets.toml` and rerun it. +- **Private key or secrets cannot be read:** verify the mounted file paths and + UID `65532` read permissions. On Linux with enforcing SELinux, also verify + that policy permits the `.dlt` bind mount. The container key path is + different from the path of the downloaded key on your host. +- **BHE authentication fails:** check the tenant URL and the matching GitHub + OpenHound collector client's token ID/key. +- **Scheduler is idle:** start an on-demand scan or schedule a collection in BHE. +- **GitHub authentication fails:** confirm the key belongs to the configured + app and that the app is installed in the enterprise and target organizations. diff --git a/docker-compose/check_config.py b/docker-compose/check_config.py new file mode 100644 index 0000000..58f6309 --- /dev/null +++ b/docker-compose/check_config.py @@ -0,0 +1,54 @@ +"""Validate local configuration without contacting GitHub or BHE.""" + +import importlib.metadata +import os +import tempfile +from pathlib import Path +from urllib.parse import urlparse + +import dlt +from dlt.common.configuration import resolve_configuration +from joserfc.jwk import RSAKey +from openhound_github.source import GithubEnterpriseAppCredentials + + +def required(name): + value = dlt.secrets.get(name) + if value is None or not str(value).strip() or "REPLACE_WITH" in str(value): + raise ValueError(f"Set {name} in .dlt/secrets.toml") + return str(value) + + +url = required("destination.bloodhoundenterprise.url") +if urlparse(url).scheme != "https" or not urlparse(url).hostname: + raise ValueError("Set a valid HTTPS BHE tenant URL") +required("destination.bloodhoundenterprise.token_id") +required("destination.bloodhoundenterprise.token_key") +required("sources.source.github.credentials.enterprise_name") + +credentials = resolve_configuration( + GithubEnterpriseAppCredentials(), + sections=("sources", "source", "github", "credentials"), +) +issuer = credentials.client_id or credentials.app_id +if not issuer or "REPLACE_WITH" in str(issuer): + raise ValueError("Set the GitHub App client_id or app_id") +key_path = Path(credentials.key_path) +key = RSAKey.import_key(key_path.read_bytes()) +if not key.is_private: + raise ValueError("The GitHub App PEM must contain a private key") + +for directory in ( + Path.cwd(), + Path(os.environ["DLT_DATA_DIR"]), + Path(os.environ["PIPELINES_DIR"]), + Path(os.environ["RUNTIME__LOG_PATH"]), + Path("/tmp/openhound"), +): + with tempfile.TemporaryFile(mode="w+", dir=directory) as probe: + probe.write("ok") + +print("PASS: BHE and enterprise-app configuration resolves; RSA private key is readable.") +print("PASS: collection, state, lookup, and log directories are writable.") +print("GitHub collector version:", importlib.metadata.version("openhound-github")) +print("Preflight does not verify credentials against GitHub or BHE.") diff --git a/docker-compose/compose.yaml b/docker-compose/compose.yaml new file mode 100644 index 0000000..2c294ed --- /dev/null +++ b/docker-compose/compose.yaml @@ -0,0 +1,62 @@ +name: openhound-github + +services: + storage-init: + image: ${OPENHOUND_IMAGE:-ghcr.io/specterops/openhound-github:latest} + user: "0:0" + network_mode: none + restart: "no" + entrypoint: ["python", "-c"] + command: + - | + import os + from pathlib import Path + for name in ("/var/lib/openhound", "/var/lib/openhound/work", "/var/lib/openhound/dlt", "/var/lib/openhound/dlt/pipelines", "/var/lib/openhound/logs", "/tmp/openhound"): + path = Path(name) + path.mkdir(parents=True, exist_ok=True) + os.chown(path, 65532, 65532) + print("Collector storage is ready.") + volumes: + - collector-data:/var/lib/openhound + - raw-data:/tmp/openhound + + scheduler-github: + image: ${OPENHOUND_IMAGE:-ghcr.io/specterops/openhound-github:latest} + init: true + restart: unless-stopped + stop_grace_period: 45s + depends_on: + storage-init: + condition: service_completed_successfully + entrypoint: ["python", "/app/src/scheduler.py"] + working_dir: /var/lib/openhound/work + environment: + DLT_PROJECT_DIR: /app + DLT_DATA_DIR: /var/lib/openhound/dlt + PIPELINES_DIR: /var/lib/openhound/dlt/pipelines + RUNTIME__LOG_PATH: /var/lib/openhound/logs + DESTINATION__BLOODHOUNDENTERPRISE__COLLECTOR_NAME: github + volumes: + - type: bind + source: ./.dlt + target: /app/.dlt + read_only: true + bind: + create_host_path: false + - type: bind + source: ./check_config.py + target: /opt/openhound-check-config.py + read_only: true + bind: + create_host_path: false + - collector-data:/var/lib/openhound + - raw-data:/tmp/openhound + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + +volumes: + collector-data: + raw-data: From e81f501c56f52dac92679578c34e3468a3521fce Mon Sep 17 00:00:00 2001 From: Jared Atkinson Date: Fri, 23 Oct 2026 14:14:57 -0700 Subject: [PATCH 2/2] fix: BED-10186 protect latest image aliases --- .github/workflows/publish-container.yml | 31 ++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish-container.yml b/.github/workflows/publish-container.yml index 074f498..39d925a 100644 --- a/.github/workflows/publish-container.yml +++ b/.github/workflows/publish-container.yml @@ -119,12 +119,37 @@ jobs: platforms: linux/amd64,linux/arm64 push: true sbom: true - tags: | - ${{ steps.image.outputs.tag }} - ${{ steps.image.outputs.latest_tag }} + tags: ${{ steps.image.outputs.tag }} labels: | org.opencontainers.image.source=https://github.com/${{ github.repository }} org.opencontainers.image.version=${{ steps.image.outputs.version }} build-args: | COLLECTOR_VERSION=${{ steps.image.outputs.version }} cache-from: type=gha,scope=${{ matrix.target }} + + - name: Update latest alias for newest release + env: + RELEASE_TAG: ${{ inputs.tag }} + VERSIONED_TAG: ${{ steps.image.outputs.tag }} + LATEST_TAG: ${{ steps.image.outputs.latest_tag }} + run: | + NEWEST_RELEASE_TAG=$( + git ls-remote --refs --tags origin 'refs/tags/v*' | + awk '$2 ~ /^refs\/tags\/v[0-9]+\.[0-9]+\.[0-9]+$/ { + sub("^refs/tags/", "", $2) + print $2 + }' | + sort -V | + tail -n 1 + ) + if [[ -z "$NEWEST_RELEASE_TAG" ]]; then + echo "Unable to determine the newest release tag" >&2 + exit 1 + fi + if [[ "$RELEASE_TAG" != "$NEWEST_RELEASE_TAG" ]]; then + echo "Skipping $LATEST_TAG: newest release is $NEWEST_RELEASE_TAG" + exit 0 + fi + docker buildx imagetools create \ + --tag "$LATEST_TAG" \ + "$VERSIONED_TAG"