diff --git a/backend/otel-collector-config.yaml b/backend/otel-collector-config.yaml index bb429ce7..ba729430 100644 --- a/backend/otel-collector-config.yaml +++ b/backend/otel-collector-config.yaml @@ -857,9 +857,9 @@ processors: - 'set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_number == SEVERITY_NUMBER_UNSPECIFIED and (IsMatch(log.body, "(?i)\"level\":\\s*\"(error|err|fatal|critical|crit|panic|dpanic)\"|\\blevel=\"?(error|err|fatal|critical|crit|panic)\\b|\\t(error|fatal|panic|dpanic)\\t|\\[(error|crit|alert|emerg)\\]") or IsMatch(log.body, "\"s\":\\s*\"[FE]\""))' - 'set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_number == SEVERITY_NUMBER_UNSPECIFIED and (IsMatch(log.body, "(?i)\"level\":\\s*\"warn(ing)?\"|\\blevel=\"?warn(ing)?\\b|\\twarn(ing)?\\t|\\[warn\\]") or IsMatch(log.body, "\"s\":\\s*\"W\""))' - 'set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_number == SEVERITY_NUMBER_UNSPECIFIED and (IsMatch(log.body, "(?i)\"level\":\\s*\"(info|debug|trace)\"|\\blevel=\"?(info|debug|trace)\\b|\\t(info|debug)\\t|\\[(notice|info|debug)\\]") or IsMatch(log.body, "\"s\":\\s*\"(I|D[1-5]?)\""))' - # Then a level word (log4j, gunicorn, Python), a traceback's first line, and Redis's marker after the time (# warning, * - . info) + # Then a level word (log4j, gunicorn, Python), a traceback's first line, Redis's marker after the time (# warning, * - . info), and the OTel SDK's export retry, a warning it prints without a level - 'set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_number == SEVERITY_NUMBER_UNSPECIFIED and IsMatch(log.body, "\\b(ERROR|FATAL|CRITICAL)\\b|^Traceback \\(most recent call last\\)")' - - 'set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_number == SEVERITY_NUMBER_UNSPECIFIED and IsMatch(log.body, "\\bWARN(ING)?\\b|^\\d+:[XCSM] \\d+ \\w+ \\d+ [\\d:.]+ # ")' + - 'set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_number == SEVERITY_NUMBER_UNSPECIFIED and IsMatch(log.body, "\\bWARN(ING)?\\b|^\\d+:[XCSM] \\d+ \\w+ \\d+ [\\d:.]+ # |^Transient error .* retrying")' - 'set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_number == SEVERITY_NUMBER_UNSPECIFIED and IsMatch(log.body, "\\b(INFO|DEBUG)\\b|^\\d+:[XCSM] \\d+ \\w+ \\d+ [\\d:.]+ [*.-] ")' # Last, a plain word in a line with no level of its own - 'set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_number == SEVERITY_NUMBER_UNSPECIFIED and IsMatch(log.body, "(?i)\\b(error|fatal|panic|exception)\\b")' diff --git a/docker-compose.yaml b/docker-compose.yaml index 1858a5c1..ec222d27 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -322,6 +322,8 @@ services: container_name: jaeger profiles: ["observability"] mem_limit: 256m + # The built-in all-in-one config keeps 100 000 traces in memory, about 260 MiB at ~2.3 KB each; 50 000 is about 4 days + command: ["--set=extensions.jaeger_storage.backends.some_storage.memory.max_traces=50000"] # OTLP 4317/4318 stays internal on app-network ports: - "127.0.0.1:6831:6831/udp" # Jaeger Thrift compact diff --git a/docs/operations/deployment.md b/docs/operations/deployment.md index 176746aa..a14d9e22 100644 --- a/docs/operations/deployment.md +++ b/docs/operations/deployment.md @@ -78,7 +78,7 @@ Every long-running service except Kafdrop has a `mem_limit`. | Frontend | 128m | | | Worker | 256m | idles near 150 MiB | | Grafana | 512m | Grafana 13 idles near 300 MiB | -| Jaeger | 256m | in-memory storage | +| Jaeger | 256m | in-memory storage, at most 50 000 traces | | Loki | 256m | | | Victoria Metrics | 256m | `--retentionPeriod=30d` | | OTel Collector | 192m | `memory_limiter` at 150 MiB | @@ -99,6 +99,19 @@ Grafana starts through a small entrypoint. It resets the admin password from `GF The frontend gets `GRAFANA_PUBLIC_URL` from `GRAFANA_ROOT_URL` (default `http://localhost:3000/`), which is where `/grafana` on the app host redirects. +### Retention and sampling + +How long each store keeps its data, and how much of it gets there, is set in the container that holds it: + +| What | Where | Value | +|---|---|---| +| Metrics | `--retentionPeriod` of `victoria-metrics` in docker-compose.yaml | 30 days | +| Logs | `retention_period` in backend/loki/loki-config.yaml | 720h, 30 days | +| Traces | `max_traces` in the `--set` flag of `jaeger` in docker-compose.yaml | 50 000, about 4 days at 12 600 a day; in memory, so a restart empties it | +| Trace sampling | `tail_sampling` in backend/otel-collector-config.yaml | every trace with an error, every trace over 1 s, 10% of the rest | + +Jaeger's built-in config would keep 100 000 traces. At about 2.3 KB each in memory that's around 260 MiB, over its 256m limit, so it would be OOM-killed after about a week and lose them all. The backend and the worker send every span (`TRACING_SAMPLING_RATE = 1.0` in config.toml) because the collector makes the app's metrics from spans; sampling happens only on the way to Jaeger. A change takes effect at the next release: compose recreates `victoria-metrics` and `jaeger` when their command changes, and the deploy restarts `loki` and `otel-collector` when their config file changes. + ## Image versions | Image | Version | diff --git a/docs/operations/tracing.md b/docs/operations/tracing.md index 1543c82f..66e59be9 100644 --- a/docs/operations/tracing.md +++ b/docs/operations/tracing.md @@ -61,6 +61,6 @@ The API stores the event in MongoDB, publishes `execution_requested` and answers ## Jaeger -Jaeger runs as `jaegertracing/jaeger:2.20.0` under the `observability` profile, with in-memory storage and the UI on `http://localhost:16686`. OTLP ports 4317 and 4318 stay on the compose network. Grafana also has a Jaeger datasource. Jaeger is pinned below 2.21 because 2.21 removed the search API that datasource calls ([grafana/grafana#112336](https://github.com/grafana/grafana/issues/112336)), so trace search in Grafana would stop working. +Jaeger runs as `jaegertracing/jaeger:2.20.0` under the `observability` profile, with in-memory storage capped at 50 000 traces (see [Retention and sampling](deployment.md#retention-and-sampling)) and the UI on `http://localhost:16686`. OTLP ports 4317 and 4318 stay on the compose network. Grafana also has a Jaeger datasource. Jaeger is pinned below 2.21 because 2.21 removed the search API that datasource calls ([grafana/grafana#112336](https://github.com/grafana/grafana/issues/112336)), so trace search in Grafana would stop working. To turn tracing off, set `TRACING_SAMPLING_RATE = 0`. Leaving `OTLP_TRACES_ENDPOINT` empty only stops the export; spans are still recorded in process, so log lines keep their `trace_id`.