feat: add openhound schema upload to install extension schemas

README.md

@@ -52,6 +52,66 @@ Follow the docs for setup, CLI usage, and collector development:
 
 - **Convert**: The raw resources are read from disk and converted to OpenGraph nodes and edges.
 
+## Uploading to BloodHound
+
+BloodHound needs two things from a collector: the extension **schema**, which describes the node and
+relationship kinds, and the **collection**, which is the data itself. OpenHound can send both.
+
+### Configure where BloodHound is
+
+Put the BloodHound address in `.dlt/config.toml` and the credentials in `.dlt/secrets.toml`, under the
+`destination.bloodhound` section. This is the same section the saved-search and privilege-zone uploads use.
+Use an API token pair (created under *My Profile → API Key Management* in BloodHound) or a bearer token from
+a signed-in session.
+
+```toml
+# .dlt/config.toml
+[destination.bloodhound]
+url = "http://localhost:8080"
+```
+
+```toml
+# .dlt/secrets.toml, with an API token pair...
+[destination.bloodhound]
+token_id = "..."
+token_key = "..."
+
+# ...or with a bearer token instead
+[destination.bloodhound]
+token = "..."
+```
+
+As environment variables these are `DESTINATION__BLOODHOUND__URL`, `DESTINATION__BLOODHOUND__TOKEN_ID`,
+`DESTINATION__BLOODHOUND__TOKEN_KEY`, and `DESTINATION__BLOODHOUND__TOKEN`. The token pair is used when both
+halves are present; otherwise the bearer token is used.
+
+Enterprise users need nothing new: when `destination.bloodhound` holds no credentials, the upload commands
+read the `destination.bloodhoundenterprise` section that Enterprise ingest already uses.
+
+### Install or update a schema
+
+```console
+# One schema file, or a zip bundle containing schema.json
+openhound schema upload ./schema.json
+openhound schema upload ./my-extension-bundle.zip
+
+# Every .json and .zip file in a folder
+openhound schema upload ./schemas
+
+# The schema.json that an installed collector ships next to its extension.yaml
+openhound schema upload sccm
+
+# Check the files without contacting BloodHound
+openhound schema upload ./schemas --dry-run
+```
+
+The command prints one line per file (`created`, `updated`, `skipped`, or `failed`) and exits with
+status 1 when any file failed. BloodHound matches extensions by the `schema.name` field, so uploading a
+changed file with the same name updates the existing extension.
+
+Collectors that want `openhound schema upload <name>` to work should ship their schema as `schema.json`
+in the same package folder as `extension.yaml`.
+
 ## Available extensions
 
 Extend OpenHound with pre-built extensions for other services. Additional collectors can be installed

src/openhound/cli/schema.py

@@ -0,0 +1,82 @@
+"""The ``openhound schema`` command group: install extension schemas into BloodHound."""
+
+import logging
+
+import typer
+
+from openhound.core.clients.from_secrets import (
+    MissingBloodHoundCredentials,
+    bloodhound_client_from_secrets,
+)
+from openhound.core.schema_upload import (
+    SchemaNotFound,
+    find_schema_files,
+    upload_schema_files,
+)
+
+logger = logging.getLogger(__name__)
+
+schema = typer.Typer(help="OpenGraph extension schema management")
+
+
+@schema.callback()
+def schema_group():
+    """Manage OpenGraph extension schemas in BloodHound.
+
+    This callback exists so that ``schema`` behaves as a command group with named
+    subcommands, such as ``openhound schema upload``, even while it has only one. Without
+    it, Typer would collapse the group and the subcommand name would be lost.
+    """
+
+
+@schema.command(help="Install or update extension schemas in BloodHound")
+def upload(
+    target: str = typer.Argument(
+        ...,
+        help=(
+            "A schema file (.json or .zip bundle), a folder of schema files, "
+            "or the name of an installed collector that ships a schema.json"
+        ),
+    ),
+    dry_run: bool = typer.Option(
+        False,
+        "--dry-run",
+        help="Check the files and show what would be uploaded without contacting BloodHound",
+    ),
+):
+    """Find the schema files, upload each one, and print a line per file.
+
+    The command exits with status 1 when any file failed, so scripts can rely on it.
+    """
+    try:
+        files = find_schema_files(target)
+    except SchemaNotFound as error:
+        typer.secho(str(error), fg=typer.colors.RED, err=True)
+        raise typer.Exit(code=1)
+
+    if dry_run:
+        # A dry run never needs credentials, so a missing configuration must not stop it.
+        client = None
+    else:
+        try:
+            client = bloodhound_client_from_secrets()
+        except MissingBloodHoundCredentials as error:
+            typer.secho(str(error), fg=typer.colors.RED, err=True)
+            raise typer.Exit(code=1)
+
+    results = upload_schema_files(client=client, files=files, dry_run=dry_run)  # type: ignore[arg-type]
+
+    any_failed = False
+    for result in results:
+        if result.succeeded:
+            typer.secho(f"{result.outcome:>8}  {result.path}", fg=typer.colors.GREEN)
+        else:
+            any_failed = True
+            typer.secho(
+                f"{result.outcome:>8}  {result.path}: {result.detail}",
+                fg=typer.colors.RED,
+                err=True,
+            )
+
+    if any_failed:
+        raise typer.Exit(code=1)

src/openhound/core/clients/bloodhound.py

@@ -24,6 +24,12 @@
 
 logger = logging.getLogger(__name__)
 
+# Seconds to wait for BloodHound when installing an extension schema: the first number is for
+# opening the connection, the second for each wait on the reply. Without them a stalled server
+# would block `schema upload` forever.
+EXTENSION_UPLOAD_CONNECT_TIMEOUT_SECONDS = 10
+EXTENSION_UPLOAD_READ_TIMEOUT_SECONDS = 120
+
 
 class BloodHoundHTTPError(DltException):
     def __init__(self, reason: str, code: int):
@@ -134,6 +140,36 @@ def custom_node(self, body: str) -> CustomNodes:
         response = self.request(method="POST", path=path, body=body.encode())
         return CustomNodes.model_validate(response.json())
 
+    def upsert_extension(self, body: bytes, content_type: str) -> bool:
+        """Install a new OpenGraph extension schema, or update the one with the same name.
+
+        BloodHound calls this operation an "upsert": it creates the extension when the
+        schema name is new and updates it when the name already exists. The request body
+        is either the schema as JSON, or a zip bundle that contains a ``schema.json`` file
+        and optional ``saved_queries.json`` and ``pz_rules.json`` files. The content type
+        tells BloodHound which of the two it is receiving.
+
+        Args:
+            body (bytes): The raw file contents to send.
+            content_type (str): Either ``application/json`` or ``application/zip``.
+
+        Returns:
+            bool: True when an existing extension was updated, False when a new one was created.
+        """
+        path = "/api/v2/extensions"
+        response = self.request(
+            method="PUT",
+            path=path,
+            body=body,
+            extra_headers={"Content-Type": content_type},
+            timeout=(
+                EXTENSION_UPLOAD_CONNECT_TIMEOUT_SECONDS,
+                EXTENSION_UPLOAD_READ_TIMEOUT_SECONDS,
+            ),
+        )
+        # BloodHound answers 200 (OK) for an update and 201 (Created) for a brand new extension.
+        return response.status_code == 200
+
     @property
     def asset_group_tags(self) -> AssetGroupsTags:
         path = "/api/v2/asset-group-tags"

src/openhound/core/clients/from_secrets.py

@@ -0,0 +1,159 @@
+"""Build a BloodHound API client from the settings stored in OpenHound's configuration.
+
+OpenHound keeps settings in dlt's two stores. Plain settings such as the BloodHound address
+live in ``.dlt/config.toml``; credentials live in ``.dlt/secrets.toml``. Both can also be
+given as environment variables. This module looks in two sections, in this order, and uses
+the first one that holds credentials:
+
+``[destination.bloodhound]``
+    The section that OpenHound's saved-search and privilege-zone uploads already use. It is
+    the home for BloodHound Community Edition settings. Holds ``url`` plus either an API token
+    pair (``token_id`` and ``token_key``, created under "My Profile > API Key Management") or
+    a bearer ``token`` from a signed-in session.
+
+``[destination.bloodhoundenterprise]``
+    The section that OpenHound's Enterprise ingest and scheduler already use. Holds ``url``
+    plus ``token_id`` and ``token_key``. Nothing about it changes; it is read here so that an
+    Enterprise user needs no extra configuration.
+
+Example, with the address in ``config.toml``::
+
+    [destination.bloodhound]
+    url = "http://localhost:8080"
+
+and the credentials in ``secrets.toml``::
+
+    [destination.bloodhound]
+    token_id = "..."
+    token_key = "..."
+
+As environment variables the same settings are ``DESTINATION__BLOODHOUND__URL``,
+``DESTINATION__BLOODHOUND__TOKEN_ID``, and so on.
+"""
+
+from dataclasses import dataclass
+
+import dlt
+
+from openhound.core.clients.bloodhound import (
+    BloodHound,
+    BloodHoundClient,
+    BloodHoundJWT,
+)
+
+# The configuration sections that may hold BloodHound settings, in the order they are tried.
+SECTION_BLOODHOUND = "destination.bloodhound"
+SECTION_ENTERPRISE = "destination.bloodhoundenterprise"
+SECTIONS_IN_ORDER = (SECTION_BLOODHOUND, SECTION_ENTERPRISE)
+
+# The setting names inside each section.
+SETTING_URL = "url"
+SETTING_TOKEN_ID = "token_id"
+SETTING_TOKEN_KEY = "token_key"
+SETTING_BEARER_TOKEN = "token"
+
+
+class MissingBloodHoundCredentials(Exception):
+    """Raised when the configuration does not contain enough information to reach BloodHound."""
+
+
+@dataclass
+class BloodHoundSettings:
+    """The settings found in one configuration section.
+
+    Attributes:
+        section: The section they came from, for example ``destination.bloodhound``.
+        url: The BloodHound address, or None when absent.
+        token_id: First half of an API token pair, or None.
+        token_key: Second half of an API token pair, or None.
+        bearer_token: A session token, or None.
+    """
+
+    section: str
+    url: str | None
+    token_id: str | None
+    token_key: str | None
+    bearer_token: str | None
+
+    @property
+    def has_token_pair(self) -> bool:
+        return bool(self.token_id and self.token_key)
+
+    @property
+    def has_credentials(self) -> bool:
+        return self.has_token_pair or bool(self.bearer_token)
+
+
+def read_settings(section: str) -> BloodHoundSettings:
+    """Read one section's settings from the two stores.
+
+    The address is read through the configuration store because that store sees
+    ``config.toml``, ``secrets.toml``, and environment variables. Credentials are read through
+    the secrets store, which sees ``secrets.toml`` and environment variables only, so a token
+    accidentally placed in ``config.toml`` is not picked up.
+    """
+    url: str | None = dlt.config.get(f"{section}.{SETTING_URL}")
+    token_id: str | None = dlt.secrets.get(f"{section}.{SETTING_TOKEN_ID}")
+    token_key: str | None = dlt.secrets.get(f"{section}.{SETTING_TOKEN_KEY}")
+    bearer_token: str | None = dlt.secrets.get(f"{section}.{SETTING_BEARER_TOKEN}")
+    return BloodHoundSettings(
+        section=section,
+        url=url,
+        token_id=token_id,
+        token_key=token_key,
+        bearer_token=bearer_token,
+    )
+
+
+def bloodhound_client_from_secrets() -> BloodHoundClient:
+    """Create the right kind of BloodHound client from whichever section holds credentials.
+
+    The sections are tried in the order of ``SECTIONS_IN_ORDER``. Within a section the API
+    token pair is preferred, because API tokens do not expire the way session tokens do. When
+    no section holds credentials, or the chosen section has no address, a plain-English error
+    names exactly which settings are needed.
+
+    Returns:
+        BloodHoundClient: A client that signs every request with the configured credentials.
+    """
+    for section in SECTIONS_IN_ORDER:
+        settings = read_settings(section)
+        if not settings.has_credentials:
+            continue
+
+        if not settings.url:
+            raise MissingBloodHoundCredentials(
+                f"Credentials were found in [{section}] but it has no 'url'. Add "
+                f"'{section}.url' to .dlt/config.toml, or set the "
+                f"{environment_variable_name(section, SETTING_URL)} environment variable."
+            )
+
+        if settings.has_token_pair:
+            # The checks above guarantee both halves are present; the assertions tell the
+            # type checker the same thing.
+            assert settings.token_id is not None and settings.token_key is not None
+            return BloodHound(
+                token_key=settings.token_key,
+                token_id=settings.token_id,
+                bhe_uri=settings.url,
+            )
+
+        assert settings.bearer_token is not None
+        return BloodHoundJWT(token=settings.bearer_token, base_uri=settings.url)
+
+    raise MissingBloodHoundCredentials(
+        "No BloodHound credentials are configured. Add a "
+        f"[{SECTION_BLOODHOUND}] section with '{SETTING_URL}' in .dlt/config.toml and either "
+        f"'{SETTING_TOKEN_ID}' and '{SETTING_TOKEN_KEY}' or '{SETTING_BEARER_TOKEN}' in "
+        f".dlt/secrets.toml. An existing [{SECTION_ENTERPRISE}] section is also accepted. "
+        "The same settings can be environment variables such as "
+        f"{environment_variable_name(SECTION_BLOODHOUND, SETTING_URL)}."
+    )
+
+
+def environment_variable_name(section: str, setting: str) -> str:
+    """Turn ``destination.bloodhound`` and ``url`` into ``DESTINATION__BLOODHOUND__URL``.
+
+    This is dlt's naming rule: dots become double underscores and everything is upper case.
+    """
+    return f"{section}.{setting}".replace(".", "__").upper()