[DX-5122] grafana alertcheck

.github/workflows/test.yaml

@@ -37,6 +37,9 @@ jobs:
           - path: parrot
             vm: ubuntu-latest
             regex: ./...
+          - path: grafana-alertcheck
+            vm: ubuntu-latest
+            regex: ./...
           - path: tools/workflowresultparser
             vm: ubuntu-latest
             regex: ./...

.gitignore

@@ -84,4 +84,6 @@ parrot/parrot
 # Devenv (generated manually)
 devenv/
 # Generated TOML definitions
-book/src/framework/developer_environment/**.toml
+book/src/framework/developer_environment/**.toml
+
+tmp/

grafana-alertcheck/.tool-versions

@@ -0,0 +1,3 @@
+# golangci-lint: keep in sync with devbox.json (used by CI in .github/workflows/linters.yml via `devbox run -- just lint`).
+golang 1.26.6
+golangci-lint 2.12.2

grafana-alertcheck/README.md

@@ -0,0 +1,46 @@
+# grafana-alertcheck
+
+A CD quality gate for Grafana alerts. It bookends a release with two commands — `watch` (record) and
+`check` (classify) — and answers whether any watched alert was in a bad state during the release window.
+
+```
+watch  → your work → check
+```
+
+`watch` starts a background recorder that polls each named alert into a JSONL log. After the work emits a
+`from`/`to` pair, `check` proves continuous coverage of that window, classifies each alert's state
+timeline, and exits `0`, `1`, or `2`.
+
+It **fails closed**: if it cannot get an answer, it stops the release — never a pass on an unproven window.
+
+## Quickstart
+
+```bash
+export GRAFANA_URL=https://grafana.example.com
+export GRAFANA_TOKEN=…
+
+grafana-alertcheck watch --out /tmp/run.jsonl --alerts alerts.txt
+./deploy.sh   # emits deployed_at=<RFC3339>
+./verify.sh   # emits finished_at=<RFC3339>
+grafana-alertcheck check --in /tmp/run.jsonl --from "$deployed_at" --to "$finished_at"
+```
+
+Requires Grafana >= 13.0.0 and < 14.0.0. Connection details come from the environment only — the token is
+never a flag.
+
+## Documentation
+
+| Doc | Covers |
+| --- | ------ |
+| [`docs/index.md`](./docs/index.md) | Overview, quickstarts, exit codes, common surprises |
+| [`docs/how-alerts-are-evaluated.md`](./docs/how-alerts-are-evaluated.md) | Verdict model, coverage proof, health/liveness |
+| [`docs/advanced.md`](./docs/advanced.md) | Check budget, scheduling, why history isn't queried |
+| [`docs/architecture.md`](./docs/architecture.md) | Design invariants, the pure-function seam, recorder lifecycle |
+| [`docs/reference/cli.md`](./docs/reference/cli.md) | Full CLI reference — subcommands, flags, naming |
+| [`docs/reference/log-format.md`](./docs/reference/log-format.md) | The JSONL log schema, for debugging artifacts |
+
+## Build
+
+```bash
+go build ./... && go test ./...
+```

grafana-alertcheck/cmd/check.go

@@ -0,0 +1,154 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/internal/gate"
+)
+
+const checkUsage = "usage: grafana-alertcheck check [--in <file>] [--pidfile F] --from RFC3339 --to RFC3339 " +
+	"[--alerts ...] [--folder F] [--states ...] [--preexisting ...] [--min-observed N] [--allow-paused] " +
+	"[--nodata-is-unobservable] [--concurrency N] [--output json]"
+
+// runCheck is the classify step's CLI surface: parse flags into a gate.Config,
+// run gate.Check, and translate its (Result, error) into output and an exit
+// code. All of the correctness lives in the gate package — this file's only job
+// is presentation and the exit-code mapping, which exitCode below keeps as one
+// pure function so it can be tested without a network.
+func runCheck(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+	fs := flag.NewFlagSet("check", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	fs.Usage = func() { fmt.Fprintln(stderr, checkUsage) }
+
+	common := registerCommon(fs)
+	in := fs.String("in", "", "path of a log recorded by watch; empty selects single-step mode")
+	pidfile := fs.String("pidfile", "", "pidfile of the recorder to stop before reading --in (default <in>.pid)")
+	from := fs.String("from", "", "the moment the deploy finished, RFC3339 (required with --in)")
+	to := fs.String("to", "", "the end of the window to classify, RFC3339 (required)")
+	states := fs.String("states", "", "comma-separated bad states to classify against (default: firing)")
+	preexisting := fs.String("preexisting", "", "how to judge an instance already bad at `from` (default: fail-unless-recovered)")
+	minObserved := fs.Int("min-observed", 0, "minimum rules that must be observed (default: every resolved rule)")
+	allowPaused := fs.Bool("allow-paused", false, "do not count a rule paused before the window against --min-observed")
+	nodataIsUnobservable := fs.Bool("nodata-is-unobservable", false, "treat a sustained health=nodata as unobservable rather than a note")
+	output := fs.String("output", "", `"json" writes the machine-readable Result to stdout in addition to the table; default is the table alone`)
+
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2
+	}
+	if fs.NArg() != 0 {
+		fmt.Fprintf(stderr, "check: unexpected arguments %v\n", fs.Args())
+		return 2
+	}
+	if *output != "" && *output != "json" {
+		fmt.Fprintf(stderr, "--output: unknown value %q (only \"json\" is supported)\n", *output)
+		return 2
+	}
+
+	url, token, err := grafanaEnv()
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 2
+	}
+	alerts, err := readAlerts(stdin, *common.alerts)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 2
+	}
+	stateList, err := parseStates(*states)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 2
+	}
+	preexistingPolicy, err := parsePreexisting(*preexisting)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 2
+	}
+
+	cfg := gate.Config{
+		URL:                  url,
+		Token:                token,
+		Alerts:               alerts,
+		Folder:               *common.folder,
+		States:               stateList,
+		Preexisting:          preexistingPolicy,
+		MinObserved:          *minObserved,
+		AllowPaused:          *allowPaused,
+		NodataIsUnobservable: *nodataIsUnobservable,
+		Log:                  *in,
+		PidFile:              *pidfile,
+		Concurrency:          *common.concurrency,
+		Clock:                gate.SystemClock{},
+		Notes:                newNoteStyler(stderr),
+	}
+	if *to == "" {
+		fmt.Fprintln(stderr, "check: --to is required")
+		return 2
+	}
+	t, err := time.Parse(time.RFC3339, *to)
+	if err != nil {
+		fmt.Fprintf(stderr, "--to: %v\n", err)
+		return 2
+	}
+	cfg.To = t
+	if *from != "" {
+		f, err := time.Parse(time.RFC3339, *from)
+		if err != nil {
+			fmt.Fprintf(stderr, "--from: %v\n", err)
+			return 2
+		}
+		cfg.From = f
+	}
+
+	// SIGINT/SIGTERM cancel the run cleanly rather than leaving an operator's
+	// Ctrl-C to kill the process mid-collection: Check's collection loop and
+	// drain wait both already select on ctx.Done() (check.go), so this makes
+	// an interrupted run fail the way every other could-not-check path does
+	// — exit 2, never a silently truncated pass.
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+
+	result, checkErr := gate.Check(ctx, cfg)
+
+	if checkErr != nil {
+		fmt.Fprintln(stderr, checkErr)
+	} else if err := renderTable(stderr, result); err != nil {
+		fmt.Fprintln(stderr, err)
+	}
+	if *output == "json" {
+		enc := json.NewEncoder(stdout)
+		enc.SetIndent("", "  ")
+		if err := enc.Encode(result); err != nil {
+			fmt.Fprintf(stderr, "encode --output json: %v\n", err)
+			return 2
+		}
+	}
+	return exitCode(result, checkErr)
+}
+
+// exitCode is the whole exit-code mapping, kept as one pure function of
+// exactly what Check returns so it is testable without a network: err != nil
+// is exit 2 UNCONDITIONALLY — never 0 and never 1, even alongside real
+// violations, because an inability to check beats a violation and an error is
+// never a pass. Violations without an error is exit 1. Neither is exit 0.
+func exitCode(res gate.Result, err error) int {
+	switch {
+	case err != nil:
+		return 2
+	case len(res.Violations) > 0:
+		return 1
+	default:
+		return 0
+	}
+}

grafana-alertcheck/cmd/check_test.go

@@ -0,0 +1,130 @@
+package main
+
+import (
+	"bytes"
+	"errors"
+	"os"
+	"testing"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/internal/gate"
+	"github.com/stretchr/testify/require"
+)
+
+// The exit-code mapping, pinned directly against exitCode with no network
+// involved: err != nil is exit 2 even alongside violations (an inability to
+// check beats a violation), violations alone are exit 1, and neither is 0.
+func TestExitCode(t *testing.T) {
+	tests := []struct {
+		name string
+		res  gate.Result
+		err  error
+		want int
+	}{
+		{"pass", gate.Result{}, nil, 0},
+		{"violation", gate.Result{Violations: []gate.Violation{{}}}, nil, 1},
+		{"error alone", gate.Result{}, errors.New("boom"), 2},
+		{"error beats violation", gate.Result{Violations: []gate.Violation{{}}}, errors.New("boom"), 2},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			require.Equal(t, tt.want, exitCode(tt.res, tt.err))
+		})
+	}
+}
+
+func writeTempAlerts(t *testing.T) string {
+	t.Helper()
+	path := t.TempDir() + "/alerts.txt"
+	require.NoError(t, os.WriteFile(path, []byte("Some Alert\n"), 0o644))
+	return path
+}
+
+// The flag-validation matrix: every one of these must fail before any network
+// call, because gate.Config.validate() runs first — an unreachable GRAFANA_URL
+// succeeding or timing out is a different test than these, which check pure
+// input validation.
+func TestRunCheck_FlagValidation(t *testing.T) {
+	tests := []struct {
+		name    string
+		env     bool
+		args    func(t *testing.T) []string
+		wantErr string
+	}{
+		{"missing env", false, func(t *testing.T) []string {
+			return []string{"--to", "2026-01-01T00:00:00Z", "--alerts", writeTempAlerts(t)}
+		}, "GRAFANA_URL"},
+		{"missing to", true, func(t *testing.T) []string {
+			return []string{"--alerts", writeTempAlerts(t)}
+		}, "--to"},
+		{"bad to", true, func(t *testing.T) []string {
+			return []string{"--to", "not-a-time", "--alerts", writeTempAlerts(t)}
+		}, "--to"},
+		{"bad from", true, func(t *testing.T) []string {
+			return []string{"--to", "2026-01-01T00:00:00Z", "--from", "not-a-time", "--alerts", writeTempAlerts(t)}
+		}, "--from"},
+		{"bad output", true, func(t *testing.T) []string {
+			return []string{"--to", "2026-01-01T00:00:00Z", "--output", "xml", "--alerts", writeTempAlerts(t)}
+		}, "--output"},
+		{"bad states", true, func(t *testing.T) []string {
+			return []string{"--to", "2026-01-01T00:00:00Z", "--states", "bogus", "--alerts", writeTempAlerts(t)}
+		}, "--states"},
+		{"states normal is rejected", true, func(t *testing.T) []string {
+			// normal is the good state, never a state to classify AS bad:
+			// accepting it would make --states normal fail every healthy
+			// instance.
+			return []string{"--to", "2026-01-01T00:00:00Z", "--states", "normal", "--alerts", writeTempAlerts(t)}
+		}, "--states"},
+		{"bad preexisting", true, func(t *testing.T) []string {
+			return []string{"--to", "2026-01-01T00:00:00Z", "--preexisting", "bogus", "--alerts", writeTempAlerts(t)}
+		}, "--preexisting"},
+		{"alerts with in", true, func(t *testing.T) []string {
+			return []string{"--to", "2026-01-01T00:00:00Z", "--in", "some.jsonl", "--alerts", writeTempAlerts(t)}
+		}, "refused"},
+		{"no alerts no in", true, func(t *testing.T) []string {
+			return []string{"--to", "2026-01-01T00:00:00Z"}
+		}, "no alert names"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.env {
+				t.Setenv("GRAFANA_URL", "http://example.invalid")
+				t.Setenv("GRAFANA_TOKEN", "test-token")
+			} else {
+				t.Setenv("GRAFANA_URL", "")
+				t.Setenv("GRAFANA_TOKEN", "")
+			}
+			var stdout, stderr bytes.Buffer
+			args := append([]string{"check"}, tt.args(t)...)
+			code := run(args, &stdout, &stderr)
+			require.Equal(t, 2, code)
+			require.Contains(t, stderr.String(), tt.wantErr)
+		})
+	}
+}
+
+// A `to` already in the past with no recorded log cannot be classified from
+// anything, because nothing ever observed the window.
+func TestRunCheck_ToInPastNoLog(t *testing.T) {
+	t.Setenv("GRAFANA_URL", "http://example.invalid")
+	t.Setenv("GRAFANA_TOKEN", "test-token")
+
+	var stdout, stderr bytes.Buffer
+	code := run([]string{"check",
+		"--from", "1999-01-01T00:00:00Z", "--to", "2000-01-01T00:00:00Z",
+		"--alerts", writeTempAlerts(t),
+	}, &stdout, &stderr)
+	require.Equal(t, 2, code)
+	require.Contains(t, stderr.String(), "already passed")
+}
+
+// --output json never writes to stdout when Check was never reached, because
+// there is no Result to encode — only the table (on stderr) can report a
+// configuration failure.
+func TestRunCheck_NoResultOnConfigError(t *testing.T) {
+	t.Setenv("GRAFANA_URL", "")
+	t.Setenv("GRAFANA_TOKEN", "")
+	var stdout, stderr bytes.Buffer
+	code := run([]string{"check", "--to", "2026-01-01T00:00:00Z", "--output", "json"}, &stdout, &stderr)
+	require.Equal(t, 2, code)
+	require.Empty(t, stdout.String())
+}