diff --git a/.github/workflows/release-deploy.yml b/.github/workflows/release-deploy.yml
index 0d3370ab..fb5fe2cb 100644
--- a/.github/workflows/release-deploy.yml
+++ b/.github/workflows/release-deploy.yml
@@ -229,6 +229,7 @@ jobs:
git diff --quiet "$PREVIOUS" HEAD -- backend/otel-host-metrics.yaml || RESTART="$RESTART host-metrics"
git diff --quiet "$PREVIOUS" HEAD -- backend/victoria-metrics || RESTART="$RESTART victoria-metrics"
git diff --quiet "$PREVIOUS" HEAD -- backend/loki || RESTART="$RESTART loki"
+ git diff --quiet "$PREVIOUS" HEAD -- backend/opensearch || RESTART="$RESTART opensearch"
git diff --quiet "$PREVIOUS" HEAD -- backend/grafana ':(exclude)backend/grafana/provisioning/dashboards' || RESTART="$RESTART grafana"
if [ -n "$RESTART" ]; then
echo "Config changed, restarting:$RESTART"
diff --git a/AGENTS.md b/AGENTS.md
index 9d089d81..99b1615a 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -10,7 +10,7 @@ Platform for running scripts online in isolated Kubernetes pods. Users write Pyt
| Backend | FastAPI, Dishka DI, Beanie ODM (MongoDB), Python 3.14 (backend/.python-version), async throughout, httpx2 for HTTP |
| Events | Kafka (KRaft, no ZooKeeper) + FastStream (Pydantic JSON) |
| Worker | one process (`app/worker.py`): execution scheduler, k8s worker, pod monitor, result processor, DLQ retries |
-| Infra | MongoDB 9, Redis 8; optional Grafana, Jaeger, Victoria Metrics, Loki, OTel Collector plus its host-metrics collector (`--observability`) and Kafdrop (`--debug`) |
+| Infra | MongoDB 9, Redis 8; optional Grafana, Jaeger, Victoria Metrics, Loki, OpenSearch, OTel Collector plus its host-metrics collector (`--observability`) and Kafdrop (`--debug`) |
| Runtimes | Python 3.11 to 3.14, Node 22/24/26, Ruby 3.3/3.4/4.0, Bash 5.3, Go 1.26/1.27 (app/runtime_registry.py) |
## Key Directory Map
@@ -81,7 +81,7 @@ npm run test:e2e # Playwright E2E, needs the running stack
./deploy.sh dev # start
./deploy.sh dev --build # rebuild and start
./deploy.sh dev --wait # start and wait for healthy
-./deploy.sh dev --observability # plus Grafana, Jaeger, Victoria Metrics, Loki, OTel Collector, host-metrics
+./deploy.sh dev --observability # plus Grafana, Jaeger, Victoria Metrics, Loki, OpenSearch, OTel Collector, host-metrics
./deploy.sh dev --debug # observability plus Kafdrop
./deploy.sh down # stop
./deploy.sh openapi # regenerate docs/reference/openapi.json
diff --git a/README.md b/README.md
index 265fda60..f2170850 100644
--- a/README.md
+++ b/README.md
@@ -87,7 +87,7 @@ curl -k https://localhost/api/v1/health/live
| Jaeger (Tracing) | [http://localhost:16686](http://localhost:16686) | needs `--observability` |
| Kafdrop (Kafka UI) | [http://localhost:9000](http://localhost:9000) | needs `--debug` |
-`./deploy.sh dev --observability` adds Grafana, Jaeger, Victoria Metrics, Loki and the OTel Collector. `--debug` adds all of those plus Kafdrop.
+`./deploy.sh dev --observability` adds Grafana, Jaeger, Victoria Metrics, Loki, OpenSearch and the OTel Collector. `--debug` adds all of those plus Kafdrop.
nginx and the backend each make a self-signed certificate when they first start, so accept the browser warning. nginx verifies the backend's certificate on every proxied request. A certificate mounted at `/etc/nginx/certs` in the frontend container is used as is.
diff --git a/backend/grafana/grafana.ini b/backend/grafana/grafana.ini
index addfd34a..41748de5 100644
--- a/backend/grafana/grafana.ini
+++ b/backend/grafana/grafana.ini
@@ -30,5 +30,5 @@ enabled = false
; Grafana 13 downloads these on first start; install them before provisioning runs
[plugins]
-preinstall_sync = prometheus,jaeger
+preinstall_sync = prometheus,jaeger,grafana-opensearch-datasource@2.34.5
; --8<-- [end:grafana_ini]
\ No newline at end of file
diff --git a/backend/grafana/provisioning/datasources/datasources.yml b/backend/grafana/provisioning/datasources/datasources.yml
index 02540562..34180635 100644
--- a/backend/grafana/provisioning/datasources/datasources.yml
+++ b/backend/grafana/provisioning/datasources/datasources.yml
@@ -37,4 +37,26 @@ datasources:
matcherType: label
matcherRegex: trace_id
name: TraceID
- url: '$${__value.raw}'
\ No newline at end of file
+ url: '$${__value.raw}'
+
+ # OpenSearch, the same logs as Loki with every field indexed; the plugin is installed when Grafana starts
+ - name: OpenSearch
+ type: grafana-opensearch-datasource
+ uid: opensearch
+ access: proxy
+ url: http://opensearch:9200
+ editable: true
+ jsonData:
+ flavor: opensearch
+ version: "3.9.0"
+ # One index a day, so a search reads only the days in its time range
+ database: "[logs-]YYYY.MM.DD"
+ interval: Daily
+ timeField: "@timestamp"
+ logMessageField: body
+ logLevelField: severity.text
+ pplEnabled: true
+ dataLinks:
+ - field: traceId
+ url: '$${__value.raw}'
+ datasourceUid: jaeger
diff --git a/backend/opensearch/logs-retention-policy.json b/backend/opensearch/logs-retention-policy.json
new file mode 100644
index 00000000..40bc0142
--- /dev/null
+++ b/backend/opensearch/logs-retention-policy.json
@@ -0,0 +1,19 @@
+{
+ "policy": {
+ "description": "Delete a daily logs index 30 days after it was created",
+ "default_state": "hot",
+ "states": [
+ {
+ "name": "hot",
+ "actions": [],
+ "transitions": [{ "state_name": "delete", "conditions": { "min_index_age": "30d" } }]
+ },
+ {
+ "name": "delete",
+ "actions": [{ "delete": {} }],
+ "transitions": []
+ }
+ ],
+ "ism_template": [{ "index_patterns": ["logs-*"], "priority": 100 }]
+ }
+}
diff --git a/backend/opensearch/logs-template.json b/backend/opensearch/logs-template.json
new file mode 100644
index 00000000..c18a1bfd
--- /dev/null
+++ b/backend/opensearch/logs-template.json
@@ -0,0 +1,17 @@
+{
+ "index_patterns": ["logs-*"],
+ "template": {
+ "settings": {
+ "number_of_shards": 1,
+ "number_of_replicas": 0
+ },
+ "mappings": {
+ "properties": {
+ "@timestamp": { "type": "date" },
+ "body": { "type": "text" },
+ "attributes": { "type": "flat_object" },
+ "resource": { "type": "flat_object" }
+ }
+ }
+ }
+}
diff --git a/backend/opensearch/start.sh b/backend/opensearch/start.sh
new file mode 100755
index 00000000..f8bee869
--- /dev/null
+++ b/backend/opensearch/start.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+set -u
+./opensearch-docker-entrypoint.sh opensearch &
+pid=$!
+trap 'kill -TERM $pid' TERM INT
+api=http://localhost:9200
+until curl -sf "$api/_cluster/health?wait_for_status=yellow&timeout=1s" > /dev/null; do
+ kill -0 $pid 2> /dev/null || exit 1
+ sleep 2
+done
+curl -sf -X PUT "$api/_index_template/logs" -H 'Content-Type: application/json' -d @/etc/opensearch-setup/logs-template.json > /dev/null || exit 1
+# A stored policy is never overwritten, so a changed policy needs a new id
+curl -sf -o /dev/null "$api/_plugins/_ism/policies/logs-30d" \
+ || curl -sf -X PUT "$api/_plugins/_ism/policies/logs-30d" -H 'Content-Type: application/json' -d @/etc/opensearch-setup/logs-retention-policy.json > /dev/null \
+ || exit 1
+# An index the collector created before the policy existed gets it too; one that has it already is skipped
+curl -s -X POST "$api/_plugins/_ism/add/logs-*" -H 'Content-Type: application/json' -d '{"policy_id": "logs-30d"}' > /dev/null
+# The first wait returns when the trap runs; the second waits for OpenSearch to finish shutting down
+wait $pid
+wait $pid
diff --git a/backend/otel-collector-config.yaml b/backend/otel-collector-config.yaml
index 4b739708..5713ac89 100644
--- a/backend/otel-collector-config.yaml
+++ b/backend/otel-collector-config.yaml
@@ -87,6 +87,9 @@ receivers:
connectors:
forward/sampling:
+ # Every log pipeline hands its lines to logs/search for OpenSearch
+ forward/search:
+
signaltometrics:
spans:
- name: queue.enqueue
@@ -843,6 +846,13 @@ processors:
log_record:
- IsMatch(log.attributes["fluent.tag"], "^(backend|worker)$") and IsMatch(log.body, "^\\{")
+ # Mongo logs every connection and login at info level, two thirds of all lines; its warnings and errors still go through
+ filter/mongo_info:
+ error_mode: ignore
+ logs:
+ log_record:
+ - log.attributes["fluent.tag"] == "mongo" and IsMatch(log.body, "\"s\":\\s*\"I\"")
+
# One resource per container, named after it, so Loki labels each container's lines with their own service_name
groupbyattrs/container:
keys: [fluent.tag]
@@ -854,6 +864,14 @@ processors:
statements:
- set(resource.attributes["service.name"], resource.attributes["fluent.tag"]) where resource.attributes["fluent.tag"] != nil
+ # structlog's JSON keys become attributes, so OpenSearch can search execution_id, user_id and the rest by name
+ transform/search:
+ error_mode: ignore
+ log_statements:
+ - context: log
+ statements:
+ - merge_maps(log.attributes, ParseJSON(log.body), "insert") where IsMatch(resource.attributes["service.name"], "^integr8scode-(backend|worker)$") and IsMatch(log.body, "^\\{")
+
# Whatever resource a browser sends, its records are labelled browser
transform/browser:
error_mode: ignore
@@ -921,6 +939,13 @@ exporters:
tls:
insecure: true
+ # One index a day, so the 30-day retention policy in backend/opensearch deletes whole days
+ opensearch:
+ http:
+ endpoint: http://opensearch:9200
+ logs_index: logs
+ logs_index_time_format: yyyy.MM.dd
+
# Console logging for debugging
debug:
verbosity: normal
@@ -970,10 +995,15 @@ service:
logs:
receivers: [otlp, fluent_forward]
- processors: [memory_limiter, filter/app_json_stdout, groupbyattrs/container, transform/container, transform/level, batch, resource]
- exporters: [otlp_http/loki, debug]
+ processors: [memory_limiter, filter/app_json_stdout, filter/mongo_info, groupbyattrs/container, transform/container, transform/level, batch, resource]
+ exporters: [otlp_http/loki, forward/search, debug]
logs/browser:
receivers: [otlp/browser]
processors: [memory_limiter, filter/browser, transform/browser, batch, resource]
- exporters: [otlp_http/loki]
\ No newline at end of file
+ exporters: [otlp_http/loki, forward/search]
+
+ logs/search:
+ receivers: [forward/search]
+ processors: [memory_limiter, transform/search]
+ exporters: [opensearch]
\ No newline at end of file
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 28e483d7..60b42da3 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -350,10 +350,40 @@ services:
command: -config.file=/etc/loki/local-config.yaml
networks:
- app-network
- mem_limit: 256m
+ mem_limit: 512m # a 30-day search took it to 207 MiB
restart: unless-stopped
# Distroless image with no shell, so no healthcheck
+ # The same lines as Loki, with every field indexed; the start script applies the logs index template and the 30-day retention policy
+ opensearch:
+ logging: *logging
+ container_name: opensearch
+ profiles: ["observability"]
+ image: opensearchproject/opensearch:3.9.0
+ entrypoint: ["bash", "/etc/opensearch-setup/start.sh"]
+ environment:
+ - discovery.type=single-node
+ # mmap would need vm.max_map_count raised on the host
+ - node.store.allow_mmap=false
+ - DISABLE_SECURITY_PLUGIN=true
+ - DISABLE_INSTALL_DEMO_CONFIG=true
+ - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m
+ ports:
+ - "127.0.0.1:9200:9200"
+ volumes:
+ - ./backend/opensearch:/etc/opensearch-setup:ro
+ - opensearch_data:/usr/share/opensearch/data
+ networks:
+ - app-network
+ mem_limit: 1536m # 30 days at prod's volume sit near 1.1 GiB with the 512m heap
+ restart: unless-stopped
+ healthcheck:
+ test: ["CMD-SHELL", "curl -sf localhost:9200/_index_template/logs > /dev/null"]
+ interval: 10s
+ timeout: 5s
+ retries: 30
+ start_period: 30s
+
# Monitoring Stack
# Victoria Metrics - Time series database
victoria-metrics:
@@ -465,6 +495,7 @@ volumes:
grafana_data:
victoria_metrics_data:
loki_data:
+ opensearch_data:
kafka_data:
kafka_logs:
# --8<-- [end:dev_volumes]
diff --git a/docs/assets/images/system_diagram.svg b/docs/assets/images/system_diagram.svg
index a85dfb36..277e0a00 100644
--- a/docs/assets/images/system_diagram.svg
+++ b/docs/assets/images/system_diagram.svg
@@ -1,4 +1,4 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/getting-started.md b/docs/getting-started.md
index a26d51dc..c4ffb675 100644
--- a/docs/getting-started.md
+++ b/docs/getting-started.md
@@ -56,7 +56,7 @@ Limits are system-wide, not per execution. An execution request carries only the
## Look around
-Kafdrop, Grafana and Jaeger aren't part of the default stack. `./deploy.sh dev --observability` adds Grafana, Jaeger, Victoria Metrics, Loki and the OTel collector, and `--debug` adds Kafdrop on top. All of them listen on localhost only.
+Kafdrop, Grafana and Jaeger aren't part of the default stack. `./deploy.sh dev --observability` adds Grafana, Jaeger, Victoria Metrics, Loki, OpenSearch and the OTel collector, and `--debug` adds Kafdrop on top. All of them listen on localhost only.
| Service | URL | Needs |
|---------|-----|-------|
diff --git a/docs/operations/deployment.md b/docs/operations/deployment.md
index a14d9e22..c5da1526 100644
--- a/docs/operations/deployment.md
+++ b/docs/operations/deployment.md
@@ -107,10 +107,11 @@ How long each store keeps its data, and how much of it gets there, is set in the
|---|---|---|
| Metrics | `--retentionPeriod` of `victoria-metrics` in docker-compose.yaml | 30 days |
| Logs | `retention_period` in backend/loki/loki-config.yaml | 720h, 30 days |
+| Searchable logs | the `logs-30d` policy in backend/opensearch/logs-retention-policy.json | 30 days, one index a day |
| Traces | `max_traces` in the `--set` flag of `jaeger` in docker-compose.yaml | 50 000, about 4 days at 12 600 a day; in memory, so a restart empties it |
| Trace sampling | `tail_sampling` in backend/otel-collector-config.yaml | every trace with an error, every trace over 1 s, 10% of the rest |
-Jaeger's built-in config would keep 100 000 traces. At about 2.3 KB each in memory that's around 260 MiB, over its 256m limit, so it would be OOM-killed after about a week and lose them all. The backend and the worker send every span (`TRACING_SAMPLING_RATE = 1.0` in config.toml) because the collector makes the app's metrics from spans; sampling happens only on the way to Jaeger. A change takes effect at the next release: compose recreates `victoria-metrics` and `jaeger` when their command changes, and the deploy restarts `loki` and `otel-collector` when their config file changes.
+Jaeger's built-in config would keep 100 000 traces. At about 2.3 KB each in memory that's around 260 MiB, over its 256m limit, so it would be OOM-killed after about a week and lose them all. The backend and the worker send every span (`TRACING_SAMPLING_RATE = 1.0` in config.toml) because the collector makes the app's metrics from spans; sampling happens only on the way to Jaeger. A change takes effect at the next release: compose recreates `victoria-metrics` and `jaeger` when their command changes, and the deploy restarts `loki`, `opensearch` and `otel-collector` when their config changes.
## Image versions
diff --git a/docs/operations/grafana-integration.md b/docs/operations/grafana-integration.md
index 478f1fdb..ab87780c 100644
--- a/docs/operations/grafana-integration.md
+++ b/docs/operations/grafana-integration.md
@@ -9,8 +9,9 @@ Grafana runs under the `observability` compose profile. Everything it shows is p
| Victoria Metrics | prometheus | `victoria-metrics` | default; exemplar `trace_id` links to Jaeger |
| Jaeger | jaeger | `jaeger` | needs Jaeger below 2.21, see [Tracing](tracing.md#jaeger) |
| Loki | loki | `loki` | `trace_id` derived field links to Jaeger |
+| OpenSearch | grafana-opensearch-datasource | `opensearch` | daily `logs-` indices; `traceId` links to Jaeger, see [OpenSearch](opensearch.md#grafana) |
-Grafana 13 downloads the prometheus and jaeger plugins on first start. `preinstall_sync` in grafana.ini makes it install them before provisioning runs, so the first start on a fresh volume needs to reach grafana.com.
+Grafana 13 downloads the prometheus, jaeger and grafana-opensearch-datasource plugins on first start. `preinstall_sync` in grafana.ini makes it install them before provisioning runs, so the first start on a fresh volume needs to reach grafana.com.
## Dashboards
diff --git a/docs/operations/loki.md b/docs/operations/loki.md
index 129fd986..6e532067 100644
--- a/docs/operations/loki.md
+++ b/docs/operations/loki.md
@@ -21,13 +21,13 @@ The OTel resource on every record has `service.name` from `SERVICE_NAME` (`integ
## Collector
-The logs pipeline receives OTLP and container output (`fluent_forward`), runs `memory_limiter`, `filter/app_json_stdout`, `groupbyattrs/container`, `transform/container`, `transform/level`, `batch` and `resource`, and exports with `otlp_http/loki` to `http://loki:3100/otlp`. The full config is [`backend/otel-collector-config.yaml`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/otel-collector-config.yaml).
+The logs pipeline receives OTLP and container output (`fluent_forward`), runs `memory_limiter`, `filter/app_json_stdout`, `filter/mongo_info`, `groupbyattrs/container`, `transform/container`, `transform/level`, `batch` and `resource`, and exports with `otlp_http/loki` to `http://loki:3100/otlp`. It also hands every line to `forward/search`, the pipeline that indexes the same lines in [OpenSearch](opensearch.md). The full config is [`backend/otel-collector-config.yaml`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/otel-collector-config.yaml).
The browser's records come in on a pipeline of their own, `logs/browser`. Its receiver, `otlp/browser`, takes OTLP over HTTP on port 4319 with bodies up to 64 KB, and only nginx's `/otlp/v1/logs` reaches it ([Frontend logging](logging.md#frontend)). The browser sends every level, and `filter/browser` is where lines get dropped: below `warning` when the collector's `ENVIRONMENT` is `production`, none anywhere else. Anyone can post to that path, so `transform/browser` drops whatever resource a record arrives with and sets `service.name` to `browser`: a posted record can't pass as the backend or any other service. The Errors dashboard and the error-lines alert group by `service_name`, so `browser` shows up there like any container.
## Container output
-Every compose service logs through Docker's `fluentd` log driver (the `x-logging` block in docker-compose.yaml) to the collector's `fluent_forward` receiver on `localhost:24224`, tagged with the container name. That covers what never reaches the Python logger: Mongo, Kafka, Redis, nginx, gunicorn's own lines and a traceback from a crash before logging starts. `groupbyattrs/container` and `transform/container` give each container its own resource with `service.name` set to the container name, so its lines carry `service_name="mongo"`, `service_name="backend"` and so on, next to `integr8scode-backend` and `integr8scode-worker` for the structured logs. `filter/app_json_stdout` drops the backend's and worker's JSON lines from stdout, since the same records arrive over OTLP with their trace ids.
+Every compose service logs through Docker's `fluentd` log driver (the `x-logging` block in docker-compose.yaml) to the collector's `fluent_forward` receiver on `localhost:24224`, tagged with the container name. That covers what never reaches the Python logger: Mongo, Kafka, Redis, nginx, gunicorn's own lines and a traceback from a crash before logging starts. `groupbyattrs/container` and `transform/container` give each container its own resource with `service.name` set to the container name, so its lines carry `service_name="mongo"`, `service_name="backend"` and so on, next to `integr8scode-backend` and `integr8scode-worker` for the structured logs. `filter/app_json_stdout` drops the backend's and worker's JSON lines from stdout, since the same records arrive over OTLP with their trace ids. `filter/mongo_info` drops Mongo's info lines (`"s":"I"`) and keeps its warnings and errors: Mongo logs every connection, its metadata and each login, and in prod those were 1.24 million of 1.87 million lines over ten days, 85% of the bytes a search had to read.
Docker's lines arrive without a level, and Loki's own guess marks Mongo's and Redis's info lines as errors, so `transform/level` gives each one from its format before Loki sees it: a JSON or logfmt `level`, Mongo's `"s"`, a zap tab-separated level, nginx's `[error]`, then a level word such as log4j's or gunicorn's `ERROR`, a traceback's first line and Redis's marker after the time, and only then a plain `error` or `warn` word. Records that already have a level, like the backend's and worker's over OTLP, keep theirs. The [Errors dashboard](grafana-integration.md#dashboards) and the error-lines alert read the result through `detected_level`.
@@ -43,7 +43,7 @@ Single instance with filesystem storage, TSDB index and schema v13. Structured m
--8<-- "backend/loki/loki-config.yaml"
```
-The container has a 256m memory limit, keeps data in the `loki_data` volume and listens on `127.0.0.1:3100`. Its image is distroless, so there's no healthcheck; Grafana and the collector start once the container has started.
+The container has a 512m memory limit (a 30-day search took it to 207 MiB), keeps data in the `loki_data` volume and listens on `127.0.0.1:3100`. Its image is distroless, so there's no healthcheck; Grafana and the collector start once the container has started.
## Queries
diff --git a/docs/operations/opensearch.md b/docs/operations/opensearch.md
new file mode 100644
index 00000000..5b322b4a
--- /dev/null
+++ b/docs/operations/opensearch.md
@@ -0,0 +1,70 @@
+# Log search (OpenSearch)
+
+OpenSearch holds the same log lines as [Loki](loki.md), with every field indexed, so a search by execution id, user id or error text over 30 days answers in milliseconds instead of reading every chunk. Loki stays for now: the plan in [#373](https://github.com/HardMax71/Integr8sCode/issues/373) is to run both on real traffic, compare, and then drop one. OpenSearch runs with the `observability` compose profile, next to Loki.
+
+```mermaid
+flowchart LR
+ Logs["logs and logs/browser pipelines"] -->|"forward/search"| Search["logs/search: transform/search"]
+ Search -->|"opensearch exporter"| OS["OpenSearch :9200, logs-YYYY.MM.DD"]
+ OS --> Grafana
+```
+
+## Collector
+
+Both log pipelines in [`backend/otel-collector-config.yaml`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/otel-collector-config.yaml) send their lines to Loki and to the `forward/search` connector, so OpenSearch gets exactly what Loki gets, after the same filters. The `logs/search` pipeline runs `transform/search`, which parses the JSON body of the backend's and worker's records into attributes (`execution_id`, `user_id`, `event` and the rest of structlog's keys), and the `opensearch` exporter writes one index a day, `logs-2026.10.22`, in the exporter's default SS4O document shape:
+
+```json
+{
+ "@timestamp": "2026-10-22T19:00:59.756Z",
+ "body": "{\"execution_id\": \"7f3c2a10-1111-4abc-9def-0123456789ab\", \"event\": \"Pod failed\", \"level\": \"error\", \"error\": \"ImagePullBackOff\"}",
+ "attributes": { "execution_id": "7f3c2a10-1111-4abc-9def-0123456789ab", "event": "Pod failed", "level": "error", "error": "ImagePullBackOff" },
+ "resource": { "service.name": "integr8scode-backend", "service.namespace": "integr8scode" },
+ "severity": { "text": "ERROR", "number": 17 },
+ "traceId": "7f08c4fb897b5f96bfa811daf69f6821",
+ "spanId": "a1b2c3d4e5f60718"
+}
+```
+
+The exporter treats a refused connection as a permanent error and doesn't retry it ([log_bulk_indexer.go#L48-L51](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/4be375e49b4d0b46733447f58e1e7a3e731ae576/exporter/opensearchexporter/log_bulk_indexer.go#L48-L51)), so lines sent while OpenSearch starts or restarts are dropped from OpenSearch. Loki still has them.
+
+## OpenSearch
+
+Single node, OpenSearch 3.9, security plugin off, listening on `127.0.0.1:9200` and on the compose network, like Loki. `node.store.allow_mmap=false` means the host's `vm.max_map_count` stays as it is. The heap is 512m and the container limit 1536m: 30 days at prod's volume sit near 1.1 GiB. Data lives in the `opensearch_data` volume, about 1.1 GB for 30 days.
+
+The container's entrypoint is [`backend/opensearch/start.sh`](https://github.com/HardMax71/Integr8sCode/blob/main/backend/opensearch/start.sh). It starts OpenSearch, waits for the cluster, and then applies two things through the API: the index template and the retention policy. The healthcheck passes only once the template exists.
+
+```json
+--8<-- "backend/opensearch/logs-template.json"
+```
+
+`attributes` and `resource` are `flat_object` fields. Every key under them is searchable by its dotted path, but they add no fields to the mapping, so a key that's a number in one line and a string in another (`status: 500`, `status: "running"`) can't make OpenSearch reject a line.
+
+```json
+--8<-- "backend/opensearch/logs-retention-policy.json"
+```
+
+The policy deletes each daily index 30 days after it was created, the same retention as Loki. A stored policy is never overwritten, so a changed policy needs a new id. An index created before the policy existed, on the very first start, gets it attached by the same script.
+
+## Grafana
+
+The datasource is the [grafana-opensearch-datasource](https://grafana.com/grafana/plugins/grafana-opensearch-datasource/) plugin, installed by `preinstall_sync` in grafana.ini. Its index is the daily pattern `[logs-]YYYY.MM.DD`, so a query reads only the days in its time range. The message field is `body`, the level field `severity.text`, and a `traceId` data link opens the trace in Jaeger.
+
+## Queries
+
+Open Grafana Explore at `http://localhost:3000/explore`, pick the OpenSearch datasource and the Logs query type, and write Lucene:
+
+```text
+# one execution, across every service
+attributes.execution_id:"7f3c2a10-1111-4abc-9def-0123456789ab"
+
+# one user's failures
+attributes.user_id:"u-42" AND severity.text:ERROR
+
+# error text anywhere in a line
+"ImagePullBackOff"
+
+# one service
+resource.service.name:"integr8scode-worker"
+```
+
+Measured locally on 1.94 million synthetic app lines spread over 31 daily indices, prod's volume once Mongo's info lines are dropped: a search by `attributes.execution_id` took 6 to 8 ms in OpenSearch (624 ms for the first query after a restart) and 25 to 75 ms through Grafana. The same search in Loki on prod's 10 days took 3.4 s, and 8.2 s with `| json`.
diff --git a/mkdocs.yml b/mkdocs.yml
index 3d196264..09f7aa41 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -165,6 +165,7 @@ nav:
- Nginx Configuration: operations/nginx-configuration.md
- Logging: operations/logging.md
- Log Aggregation (Loki): operations/loki.md
+ - Log Search (OpenSearch): operations/opensearch.md
- Tracing: operations/tracing.md
- Metrics:
- Reference: operations/metrics-reference.md