From 70ae59b72bc6df124ba19318d5bed0d6df815fa0 Mon Sep 17 00:00:00 2001 From: Cedric Cordenier Date: Wed, 9 Sep 2026 11:43:28 +0100 Subject: [PATCH] Add libs/capability: reflection-driven capability runner Adds libs/capability, an in-progress replacement for the libs/standalone bootstrapper where a capability binary's main reduces to: capability.Run(ctx, trigger.NewCron) Run reads the constructor's parameter list by reflection, builds the dependencies it asks for (logger, registry, limits factory), and hangs run/embed subcommands off a cobra root. See libs/capability/AGENTS.md. Its one in-repo dependency, libs/standalone/protohelpers/ui (the debug UI served under /debug/capabilities), comes along. The rest of the spike branch's libs (standalone rewrite, ocr, grpcutils, x) is not included. NOTE: pins chainlink-common to v0.11.2-0.20260827151019-9853beb3a544, the tip of the unmerged chainlink-common branch rtinianov_configWithOwner, because registry.Local/registry.Registry/registry.RegisterCapability do not exist on chainlink-common main yet. That branch should land first and this pin be updated to a main pseudo-version afterwards. --- libs/capability/AGENTS.md | 226 ++++ libs/capability/capability.go | 65 + libs/capability/config.go | 48 + libs/capability/debugui.go | 32 + libs/capability/debugui_test.go | 57 + libs/capability/logger.go | 22 + libs/capability/observability.go | 46 + libs/capability/observability_test.go | 98 ++ libs/capability/pyroscope.go | 93 ++ libs/capability/registry.go | 190 +++ libs/capability/registry_test.go | 172 +++ libs/capability/run.go | 225 ++++ libs/capability/run_test.go | 386 ++++++ libs/capability/runner.go | 139 +++ libs/capability/runner_test.go | 280 +++++ libs/capability/server.go | 142 +++ libs/capability/settings.go | 117 ++ libs/capability/settings_test.go | 102 ++ libs/capability/telemetry.go | 271 +++++ libs/capability/telemetry_test.go | 401 +++++++ libs/capability/webserver.go | 179 +++ libs/go.mod | 64 +- libs/go.sum | 132 ++- libs/standalone/protohelpers/ui/assets.go | 95 ++ libs/standalone/protohelpers/ui/context.go | 62 + libs/standalone/protohelpers/ui/errors.go | 102 ++ libs/standalone/protohelpers/ui/fleet.go | 99 ++ libs/standalone/protohelpers/ui/hub.go | 602 ++++++++++ libs/standalone/protohelpers/ui/hub_test.go | 574 +++++++++ libs/standalone/protohelpers/ui/metadata.go | 445 +++++++ .../protohelpers/ui/metadata_test.go | 274 +++++ libs/standalone/protohelpers/ui/mount.go | 488 ++++++++ .../protohelpers/ui/resources/form.js | 817 +++++++++++++ .../protohelpers/ui/resources/page.css | 620 ++++++++++ .../protohelpers/ui/resources/request.html | 117 ++ .../protohelpers/ui/resources/request.js | 1044 +++++++++++++++++ .../ui/resources/subscriptions.js | 546 +++++++++ .../protohelpers/ui/resources/values.js | 287 +++++ .../protohelpers/ui/response_test.go | 209 ++++ libs/standalone/protohelpers/ui/rows.go | 207 ++++ libs/standalone/protohelpers/ui/shim.go | 133 +++ libs/standalone/protohelpers/ui/shim_test.go | 80 ++ libs/standalone/protohelpers/ui/special.go | 148 +++ .../protohelpers/ui/special_test.go | 103 ++ libs/standalone/protohelpers/ui/stream.go | 219 ++++ .../protohelpers/ui/subscribe_test.go | 445 +++++++ .../protohelpers/ui/subscriptions.go | 151 +++ libs/standalone/protohelpers/ui/trigger.go | 52 + libs/standalone/protohelpers/ui/ui.go | 505 ++++++++ libs/standalone/protohelpers/ui/ui_test.go | 577 +++++++++ 50 files changed, 12424 insertions(+), 64 deletions(-) create mode 100644 libs/capability/AGENTS.md create mode 100644 libs/capability/capability.go create mode 100644 libs/capability/config.go create mode 100644 libs/capability/debugui.go create mode 100644 libs/capability/debugui_test.go create mode 100644 libs/capability/logger.go create mode 100644 libs/capability/observability.go create mode 100644 libs/capability/observability_test.go create mode 100644 libs/capability/pyroscope.go create mode 100644 libs/capability/registry.go create mode 100644 libs/capability/registry_test.go create mode 100644 libs/capability/run.go create mode 100644 libs/capability/run_test.go create mode 100644 libs/capability/runner.go create mode 100644 libs/capability/runner_test.go create mode 100644 libs/capability/server.go create mode 100644 libs/capability/settings.go create mode 100644 libs/capability/settings_test.go create mode 100644 libs/capability/telemetry.go create mode 100644 libs/capability/telemetry_test.go create mode 100644 libs/capability/webserver.go create mode 100644 libs/standalone/protohelpers/ui/assets.go create mode 100644 libs/standalone/protohelpers/ui/context.go create mode 100644 libs/standalone/protohelpers/ui/errors.go create mode 100644 libs/standalone/protohelpers/ui/fleet.go create mode 100644 libs/standalone/protohelpers/ui/hub.go create mode 100644 libs/standalone/protohelpers/ui/hub_test.go create mode 100644 libs/standalone/protohelpers/ui/metadata.go create mode 100644 libs/standalone/protohelpers/ui/metadata_test.go create mode 100644 libs/standalone/protohelpers/ui/mount.go create mode 100644 libs/standalone/protohelpers/ui/resources/form.js create mode 100644 libs/standalone/protohelpers/ui/resources/page.css create mode 100644 libs/standalone/protohelpers/ui/resources/request.html create mode 100644 libs/standalone/protohelpers/ui/resources/request.js create mode 100644 libs/standalone/protohelpers/ui/resources/subscriptions.js create mode 100644 libs/standalone/protohelpers/ui/resources/values.js create mode 100644 libs/standalone/protohelpers/ui/response_test.go create mode 100644 libs/standalone/protohelpers/ui/rows.go create mode 100644 libs/standalone/protohelpers/ui/shim.go create mode 100644 libs/standalone/protohelpers/ui/shim_test.go create mode 100644 libs/standalone/protohelpers/ui/special.go create mode 100644 libs/standalone/protohelpers/ui/special_test.go create mode 100644 libs/standalone/protohelpers/ui/stream.go create mode 100644 libs/standalone/protohelpers/ui/subscribe_test.go create mode 100644 libs/standalone/protohelpers/ui/subscriptions.go create mode 100644 libs/standalone/protohelpers/ui/trigger.go create mode 100644 libs/standalone/protohelpers/ui/ui.go create mode 100644 libs/standalone/protohelpers/ui/ui_test.go diff --git a/libs/capability/AGENTS.md b/libs/capability/AGENTS.md new file mode 100644 index 000000000..db66f58b9 --- /dev/null +++ b/libs/capability/AGENTS.md @@ -0,0 +1,226 @@ +# `capability` — standalone capability binary, spike + +## What this is + +An in-progress replacement for `libs/standalone`'s bootstrapper. The goal is that a capability +binary's `main` is one line: + +```go +func main() { capability.Run(ctx, trigger.NewCron) } +``` + +Everything a hand-written `main` does today — registering flags, resolving dependencies, handing +them to a factory — is a restatement of the capability constructor's parameter list. `Run` reads +that list by reflection and builds what it asks for. + +The reference implementation to compare against is `libs/standalone/bootstrapper.go` (process +lifecycle) and `libs/standalone/capability` (capability hosting). Neither has been changed except +for one extraction (see *Relationship to `libs/standalone`*). + +`cron/main.go` is the first user: `capability.Run(ctx, trigger.NewCron, capability.WithOtelViews(...))`, +with `trigger.Config` embedding `capability.Config`. The old bootstrapper remains for everything +else. + +## Current state + +`RunErr(ctx, lggr, ctor, opts...)` builds a cobra root (named after the executable), registers the +config sections on it, and hangs `run` and `embed` subcommands off it. `embed` is a stub. + +`run` (in `runner.go`) is the whole sequence, in order: + +``` +profiler (nil when no pyroscope server configured) +telemetry building it installs the process-global beholder client +registry grpc.NewClient (lazy) + registry.Local[.WithRemote]; Add serves+announces +settings CRE settings from the dumped file, for the limits factory +startCapability build (newCapability.call) → Start → reg.Add (serve+announce) → debug UI if flagged +health checker reports on the services started before it +web server /metrics, /debug/pprof, /healthz, /readyz, /reload/settings.txt, + /debug/capabilities when --capabilities.http-debug +→ block: plugin.Serve under a go-plugin host, else <-ctx.Done() +→ defer MultiCloser(svcs): concurrent — close order is not controlled +``` + +`run` keeps a plain `[]services.Service`: each piece is **started as it is built** (`startX` +wrappers over `newX`) and appended, and a deferred `services.MultiCloser` at the top closes +whatever was appended — so a failure at any step unwinds everything before it. There is no +aggregate root service. + +Config is `config` (`config.go`) with `observability`, `capabilities` and `grpc` as siblings. Every +setting +is optional; `--http.port` defaults to 8080. A constructor may also declare one config struct of +its own by embedding `Config`; `bindConfig` registers it on the root under the binary's name +(`--cron.fastest-schedule-interval-seconds`, `CRE_CRON_...`) and `call` hands it over decoded. + +Dependency injection: `constructor.call` (`run.go`) matches constructor parameters to +`Dependencies` fields **by type, by assignability, one field at a time** — except the `Config` +parameter, which comes from the flags. Currently `Logger`, `CapabilityRegistry` and +`LimitsFactory`. + +## The list + +Numbered against `libs/standalone/bootstrapper.go` unless noted. + +### Done +1. Block until interrupted — `signal.NotifyContext` in `RunErr`, `<-ctx.Done()` in `run` +2. `plugin.Serve` under a go-plugin host — `underPluginHost()` + empty LOOP +3. `logger.Sync()` on shutdown — deferred first so it unwinds last +4. Logger named after the binary +6. `WithOtelViews` — `Option` on `Run`/`RunErr` +7. Start the capability — an entry in the run's service list +8. Health checker registration — registers its *siblings*, not the parent +16. Serve each capability on its own gRPC server — `server.go` + `registryService.Add`. + `server.go` is a copy of `libs/standalone/grpc` minus the configured single-server form + (crecore's), with one fix: the stop happens before the engine handoff, not in the `Close` + hook, because the engine waits for the `Serve` goroutine *before* running the hook — the old + arrangement deadlocks a started server's close. +17. Register and announce to the node's registry, remove on shutdown — `registryService.Add` and + `close`. The registry owns serving itself: `Add` binds the server, holds the value, and calls + `AddAt` with the address in scope, so the `addresses`-map convention `WithRemote` expects is + deliberately nil'd out. Announcing happens at build time (right after the constructor), so a + failed announce fails the run before it is nominally up and ready implies announced. `close` + is one ordered function — Remove, stop servers, close conn — and uses a fresh context, not the + engine's: `Close` closes the `StopChan` the engine's derives from before running the hook, so + the old `eng.NewCtx()` deregistration was cancelled before it left the process (latent in + `libs/standalone/capability` too). + +### Skipped +5. `commonConfig` — an empty struct upstream; not needed yet + +### Open — bootstrapper +9. Expose the run to constructors: the mux, metrics registerer, beholder client. The mux is the + run's own (created early in `run`, so the reload endpoint registers on it), but constructors + still have no way to reach it. +10. Resolve dependencies before calling the constructor — `bootstrap_gen.go` `Run1`…`Run10` +11. Close resolved dependencies in reverse — `registerCloser` (`:150`) +12. Register each dependency's config on the command that resolves it — + `setupCommands`/`collectTargets`/`configSet` (`:374-479`). Each config instance must be bound + exactly once or viper reads the wrong flag. + +### Open — embed +13. `embed` command + `--instances` (stub at `run.go`) +14. `ForEmbedding` — per-instance dependency forms +15. Per-instance identity: logger `instance.N`, prometheus `instance` label (`:342-348`), + `portFor(index)`, and distinct service names or health metrics collide between instances + +### Done — capability hosting +18. Settings reload endpoint — `reloadHandler` in `settings.go`, registered on the run's mux in + `run`. The node dumps the file and hits `/reload/settings.txt`; 200 means every limit now + resolves against the new payload, 500 means the previous settings are still in force. +19. Debug UI — `mountDebugUI` in `debugui.go`, under `/debug/capabilities`, gated on + `--capabilities.http-debug` (off by default: it invokes capabilities). Fleet and hub are one + each; they are the shared forms an embed run fans out over, so `embed` takes them over rather + than inventing its own. + +## Decisions, and the constraints behind them + +These are the non-obvious ones. Several were re-litigated more than once before the constraint was +found, so they are worth reading before changing the order of anything in `run`. + +**Telemetry is installed at build time, not in `Start`.** A capability creates its OTEL instruments +while it is being *constructed* (`cron/trigger/metrics.go:47` calls `beholder.GetMeter()` inside +`NewMetrics`), and an instrument resolves its meter once. Installing later leaves every capability +metric bound to the noop meter for the life of the process, silently. Same reason the limits factory +is built after telemetry. + +**The health checker cannot be inside the thing it reports on.** `Register` seeds state by calling +`reporter.Ready()` immediately and only re-reads on a **15s** tick (`services/health.go:63`). +Registering a still-starting aggregate would leave `/readyz` wrong for up to 15 seconds. It +registers its siblings instead — `run` hands it a snapshot of the slice built so far, which is +exactly what is already running when it starts. + +**Constructors ask for individual dependencies, never the `Dependencies` struct.** Taking the struct +would be a dependency on everything a run has, and adding a field would silently widen what every +capability appears to need. `offered()` enforces this by not offering the struct. + +**Matching is by assignability.** A capability can ask for `core.CapabilitiesRegistry` rather than +the wider `registry.Registry` the run holds. `provide()` reflects on the *dynamic* type, so a field +declared as a narrow interface still matches wider ones. + +**The registry serves what it registers.** `registryService.Add` binds the server, mounts the +capability, holds the value locally, and announces with `AddAt` — one function, so serve-first- +announce-last is control flow rather than convention, and the address never leaves the scope that +made it. The `addresses` map `WithRemote` announces from is nil'd out: a shared write between +whoever serves and whoever announces is exactly the coupling this removes. + +**The capability is started before it is announced.** `startCapability` runs the constructor, +starts the capability, and only then `reg.Add` serves and announces it — traffic the announcement +invites lands on something already running. Announcing at build time, rather than in a service's +`Start` of its own, means a failed announce fails the run before it is nominally up, and `/readyz` +implies announced. + +**`grpc.NewClient` does not dial, and does not validate.** The node starts this process and the two +race, so an eager dial would be a race we lose intermittently. It also accepts `""`, `"!://x"` and +unknown schemes without error — so a typo in `--capabilities.proxy-url` surfaces as a failed lookup +much later, not at startup. + +**Services start eagerly, in build order, and close concurrently** (`services/multi.go` — +`MultiCloser`). `run` starts each service as it builds it (`startX` wrappers) and appends it to +the slice; the deferred `MultiCloser` at the top of `run` closes whatever was appended, which is +what makes a failure at any step unwind everything before it. Close order is *not* controlled. + +**`StopOnce` refuses to run a `Close` hook on a service that never started** +(`services/state.go:111`, `ErrCannotStopUnstarted`). This is why anything that changes process state +at build time cannot rely on `Close` alone to undo it. + +## Known gaps and defects + +- **~~Telemetry global leaks on a pre-start failure~~ — resolved.** Every service is started as it + is built and appended to `svcs`, and the deferred `MultiCloser` at the top of `run` closes + whatever was appended, so a failure at any later step — the constructor, `newSettings`, + `reg.Add` — unwinds telemetry's global swap along with everything else. +- **~~A failure after `reg.Add` leaks the announcement~~ — resolved**, same mechanism: the + registry is started (so its `Close` runs) and on the slice before `startCapability` is called. +- **`embed` must not serve the plugin host.** `run` decides for itself via `underPluginHost()`, so + every instance would try. A host supervises one plugin. `TODO` on `run`. +- **Close ordering.** The registry's own close is now internally ordered (Remove → stop servers → + close conn), so the deregistration reliably reaches the node. What remains: the registry and the + capability are still closed concurrently (`MultiCloser`), so a draining RPC (servers stop with + `GracefulStop`) can call into a capability that is already closing. The bootstrapper closed + dependencies strictly after services (`registerCloser`). Unresolved. +- **Settings constants are a copy.** `settingsDirName`, `settingsFileName`, `reloadPathPrefix` in + `settings.go` duplicate `libs/standalone/capability/settings.go` because importing it would be a + cycle. They are a live contract with the node, which writes the file this reads. If they drift, + settings silently stop arriving and every limit falls back to its compiled-in default with no + error. `TestSettingsPathIsTheSharedConvention` guards this side only. +- **`CL_PROMETHEUS_PORT` is ignored.** Under a go-plugin host the node assigns a metrics port via + that env var, but this binds `CRE_HTTP_PORT`/`CL_HTTP_PORT`. Pre-existing — the bootstrapper has + the same naming — but now quieter, since the 8080 default means the binary starts anyway. +- **`loop.TracingConfig.OnDialError` is still nil in `libs/standalone/telemetry.go`.** Fixed here + (`telemetry.go`, `beholderConfig`); still live in the shipping bootstrapper, where + `--tracing.enabled` plus an unreachable collector is a nil dereference on a background goroutine + that kills the process. + +## Relationship to `libs/standalone` + +This package is a deliberate **copy** of the observability config and helpers, not a move. +`libs/standalone` imports this package for `NewLogger`, so this package cannot import it back +without a cycle. The duplicate is meant to resolve by *deletion* — when this is what starts a +binary, the ones in `standalone` go. + +The only change made to `libs/standalone` is that `newLogger` was moved out of `telemetry.go` into +this package's `logger.go`, and `bootstrapper.go` now calls `capability.NewLogger()`. + +## Running the tests + +``` +cd libs +go test ./capability/ -short # ~1s +go test ./capability/ -short -race +go test ./capability/ # ~21s +``` + +**Use `-short` by default.** One test — +`TestRunInstallsTelemetryBeforeBuildingTheCapability` — builds a real beholder client, and closing +one flushes to a collector that is not there: about 20s of export timeouts. It is gated on +`testing.Short()`. It is also the only test that would catch the silent-noop-metrics regression +described above, so do not delete it. + +Tests bind real TCP ports (`freePort`) and mutate `os.Args` and the beholder global, so they are not +parallel-safe. `execute()` in `run_test.go` drives the real entry point and stops the run by +cancelling its context once the HTTP server answers. + +## Style + +Comments explain *why*, not what — particularly the ordering constraints above, which are otherwise +invisible and have been reintroduced by accident more than once. Match the surrounding density. diff --git a/libs/capability/capability.go b/libs/capability/capability.go new file mode 100644 index 000000000..ab7cf27a4 --- /dev/null +++ b/libs/capability/capability.go @@ -0,0 +1,65 @@ +package capability + +import ( + "reflect" + + "google.golang.org/protobuf/reflect/protoreflect" + + "github.com/smartcontractkit/chainlink-common/pkg/capabilities" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + "github.com/smartcontractkit/chainlink-common/pkg/types/core" +) + +type Capability interface { + services.Service + capabilities.ExecutableAndTriggerCapability + + // Service is the proto service this capability's server was generated from. + Service() protoreflect.ServiceDescriptor +} + +// Config marks the struct a constructor asks for as the capability's own configuration +// +// type Config struct { +// capability.Config +// FastestScheduleIntervalSeconds int `usage:"fastest cron schedule a workflow may register"` +// } +// +// The run binds the struct's fields as flags on the root command, namespaced by the binary's +// name - cron's are --cron.fastest-schedule-interval-seconds, CRE_CRON_FASTEST_SCHEDULE_INTERVAL_SECONDS, +// or the same key in the config file - and hands the constructor the decoded value. A constructor +// declares at most one. +type Config struct{} + +// capabilityConfig is what a capability binary needs from the node it runs beside. +type capabilityConfig struct { + ProxyURL string `usage:"gRPC target of the node's capability registry proxy (e.g. localhost:9000), used to resolve capabilities this binary does not host. Unset resolves only the capabilities this binary hosts"` + CapabilityDonID uint32 `usage:"on-chain DON ID of the capability DON this process was spawned for"` + + // Serves the Debug UI + HTTPDebug bool `usage:"serve the capability debug UI on the shared HTTP server, under /debug/capabilities"` +} + +type Dependencies struct { + Logger logger.Logger + CapabilityRegistry core.CapabilitiesRegistry + LimitsFactory limits.Factory +} + +func (d Dependencies) list() []any { + return []any{d.Logger, d.CapabilityRegistry, d.LimitsFactory} +} + +func (d Dependencies) resolve(want reflect.Type) (reflect.Value, bool) { + for _, v := range d.list() { + if v == nil { + continue + } + if got := reflect.TypeOf(v); got.AssignableTo(want) { + return reflect.ValueOf(v), true + } + } + return reflect.Value{}, false +} diff --git a/libs/capability/config.go b/libs/capability/config.go new file mode 100644 index 000000000..49490abfa --- /dev/null +++ b/libs/capability/config.go @@ -0,0 +1,48 @@ +package capability + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/smartcontractkit/chainlink-common/pkg/config/flags" +) + +const capabilitiesNamespace = "capabilities" + +const grpcNamespace = "grpc" + +// the standard configuration for a capability; does not include the config that a capability may request via its constructor. +type config struct { + observability observability + capabilities capabilityConfig + grpc grpcConfig +} + +func defaultConfig() *config { + return &config{ + observability: *defaultObservability(), + grpc: grpcConfig{AdvertiseHost: defaultHost}, + } +} + +// namespaced pairs every config with the namespace it is registered under, in the order the flags +// are registered. +func (c *config) namespaced() []section { + return append(c.observability.namespaced(), + section{capabilitiesNamespace, &c.capabilities}, + section{grpcNamespace, &c.grpc}, + ) +} + +// bind binds every config to root, each under the namespace that owns it. +func (c *config) bind(root *cobra.Command) error { + opts := flags.DefaultTOMLOptions("CRE", "CL") + for _, s := range c.namespaced() { + opts.Namespace = s.namespace + if err := flags.RegisterCommandFlags(root, s.target, opts); err != nil { + return fmt.Errorf("failed to register the %s settings: %w", s.namespace, err) + } + } + return nil +} diff --git a/libs/capability/debugui.go b/libs/capability/debugui.go new file mode 100644 index 000000000..bb2e57301 --- /dev/null +++ b/libs/capability/debugui.go @@ -0,0 +1,32 @@ +package capability + +import ( + "context" + "fmt" + "net/http" + + "github.com/smartcontractkit/capabilities/libs/standalone/protohelpers/ui" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" +) + +// mountDebugUI serves the debug page for the capability this binary hosts. +func mountDebugUI(ctx context.Context, lggr logger.Logger, mux *http.ServeMux, registry ui.Registry, c Capability) error { + server, err := ui.New(ctx, registry, c) + if err != nil { + return fmt.Errorf("failed to build the capability debug UI: %w", err) + } + + if err := ui.Mount(ui.Options{ + Mux: mux, + Server: server, + Fleet: &ui.Fleet{}, + Hub: ui.NewHub(), + Title: "Capability debug", + }); err != nil { + return fmt.Errorf("failed to mount the capability debug UI: %w", err) + } + + lggr.Infow("Serving the capability debug UI", "path", ui.DefaultPrefix+"/ui/", "fanout", ui.DefaultPrefix+"/request") + return nil +} diff --git a/libs/capability/debugui_test.go b/libs/capability/debugui_test.go new file mode 100644 index 000000000..8d16ee49b --- /dev/null +++ b/libs/capability/debugui_test.go @@ -0,0 +1,57 @@ +package capability + +import ( + "context" + "fmt" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" +) + +// TestRunServesTheDebugUIWhenAsked covers the opt-in: with --capabilities.http-debug the page is +// served under /debug/capabilities. +func TestRunServesTheDebugUIWhenAsked(t *testing.T) { + cfg, c, port := runArgs(t, newFake) + cfg.capabilities.HTTPDebug = true + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- run(ctx, logger.Test(t), "cron", cfg, c) }() + + require.Eventually(t, func() bool { return serving(int(port)) }, 30*time.Second, 5*time.Millisecond, + "the run should be serving while it is up") + + res, err := http.Get(fmt.Sprintf("http://localhost:%d/debug/capabilities/ui/", port)) + require.NoError(t, err) + require.NoError(t, res.Body.Close()) + assert.Equal(t, http.StatusOK, res.StatusCode) + + cancel() + require.NoError(t, <-done) +} + +// TestRunServesNoDebugUIByDefault pins the default: the UI invokes capabilities, so a run that +// was not asked for it does not expose it. +func TestRunServesNoDebugUIByDefault(t *testing.T) { + cfg, c, port := runArgs(t, newFake) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- run(ctx, logger.Test(t), "cron", cfg, c) }() + + require.Eventually(t, func() bool { return serving(int(port)) }, 30*time.Second, 5*time.Millisecond, + "the run should be serving while it is up") + + res, err := http.Get(fmt.Sprintf("http://localhost:%d/debug/capabilities/ui/", port)) + require.NoError(t, err) + require.NoError(t, res.Body.Close()) + assert.Equal(t, http.StatusNotFound, res.StatusCode) + + cancel() + require.NoError(t, <-done) +} diff --git a/libs/capability/logger.go b/libs/capability/logger.go new file mode 100644 index 000000000..c2e755b57 --- /dev/null +++ b/libs/capability/logger.go @@ -0,0 +1,22 @@ +package capability + +import ( + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" +) + +// NewLogger returns a logger encoding hclog-compatible JSON on stderr, like a +// LOOP plugin's: a go-plugin host parses and re-levels these entries, while +// standalone they are plain zap JSON logs. Level is Debug because filtering is +// the reader's job (host-side, or the log pipeline). +func NewLogger() (logger.Logger, error) { + return logger.NewWith(func(cfg *zap.Config) { + cfg.Level.SetLevel(zap.DebugLevel) + cfg.EncoderConfig.LevelKey = "@level" + cfg.EncoderConfig.MessageKey = "@message" + cfg.EncoderConfig.TimeKey = "@timestamp" + cfg.EncoderConfig.EncodeTime = zapcore.TimeEncoderOfLayout("2006-01-02T15:04:05.000000Z07:00") + }) +} diff --git a/libs/capability/observability.go b/libs/capability/observability.go new file mode 100644 index 000000000..26f704117 --- /dev/null +++ b/libs/capability/observability.go @@ -0,0 +1,46 @@ +package capability + +import "go.opentelemetry.io/otel/sdk/metric" + +const ( + telemetryNamespace = "telemetry" + tracingNamespace = "tracing" + chipIngressNamespace = "chip-ingress" + pyroscopeNamespace = "pyroscope" + httpNamespace = "http" +) + +// observability is every process-wide observability config, registered together and consumed once +// the command runs and they have been decoded. +type observability struct { + telemetry TelemetryConfig + tracing TracingConfig + chipIngress ChipIngressConfig + pyroscope PyroscopeConfig + http HTTPConfig + + otelViews []metric.View // supplied through capability.Run +} + +func defaultObservability() *observability { + return &observability{ + tracing: TracingConfig{SamplingRatio: 1}, + http: HTTPConfig{Port: defaultHTTPPort}, + } +} + +type section struct { + namespace string + target any +} + +// namespaced pairs each config with its namespace, in the order the flags are registered. +func (o *observability) namespaced() []section { + return []section{ + {telemetryNamespace, &o.telemetry}, + {tracingNamespace, &o.tracing}, + {chipIngressNamespace, &o.chipIngress}, + {pyroscopeNamespace, &o.pyroscope}, + {httpNamespace, &o.http}, + } +} diff --git a/libs/capability/observability_test.go b/libs/capability/observability_test.go new file mode 100644 index 000000000..57e3fc182 --- /dev/null +++ b/libs/capability/observability_test.go @@ -0,0 +1,98 @@ +package capability + +import ( + "fmt" + "io" + "net" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" +) + +// TestProfilerWithoutAServer is the off case, which is every binary that has not configured +// pyroscope: there is no service at all, rather than one that does nothing. +// +// Nil rather than a no-op matters to the caller: the root would take a no-op and report on it, and +// a health report mentioning profiling that is not happening is worse than not mentioning it. +func TestProfilerWithoutAServer(t *testing.T) { + assert.Nil(t, newProfiler(logger.Test(t), "cron", PyroscopeConfig{})) +} + +// TestProfilerWithAServer covers the other side: a configured one is a service the root can take, +// named so it is tellable in a health report. +// +// It is built rather than started. Starting one dials a pyroscope server, which a test has no +// business standing up to prove a constructor returned something. +func TestProfilerWithAServer(t *testing.T) { + p := newProfiler(logger.Test(t), "cron", PyroscopeConfig{ServerAddress: "pyro:4040"}) + + require.NotNil(t, p) + assert.Equal(t, "Profiler", p.Name()) +} + +// TestWebServerServes is what the HTTP config buys: the endpoints an operator and a prometheus +// scrape depend on, on the configured port, and gone again when it stops. +func TestWebServerServes(t *testing.T) { + health, err := newHealthChecker(logger.Test(t), beholder.NewNoopClient(), []services.HealthReporter{newFake()}) + require.NoError(t, err) + require.NoError(t, health.Start(t.Context())) + t.Cleanup(func() { assert.NoError(t, health.Close()) }) + checker := health.checker + + port := freePort(t) + mux := http.NewServeMux() + + // A route registered before the server is built, which is when a service registers its own. + mux.HandleFunc("/capability", func(w http.ResponseWriter, _ *http.Request) { + fmt.Fprintln(w, "hello") + }) + + web := newWebServer(logger.Test(t), HTTPConfig{Port: uint16(port)}, mux, checker) + require.NoError(t, web.Start(t.Context())) + + for path, want := range map[string]int{ + "/metrics": http.StatusOK, + "/debug/pprof/": http.StatusOK, + "/capability": http.StatusOK, + // The capability is built but not started, so it is not ready - which is the honest + // answer rather than a health check that reports on nothing. + "/readyz": http.StatusServiceUnavailable, + } { + res, err := http.Get(fmt.Sprintf("http://localhost:%d%s", port, path)) + require.NoError(t, err, path) + _, _ = io.Copy(io.Discard, res.Body) + require.NoError(t, res.Body.Close()) + assert.Equal(t, want, res.StatusCode, path) + } + + // Stopping frees the port, so the next thing to want it can have it. + require.NoError(t, web.Close()) + _, err = http.Get(fmt.Sprintf("http://localhost:%d/metrics", port)) + assert.Error(t, err, "the server should not answer once it has stopped") +} + +// TestWebServerReportsAPortItCannotHave covers the failure a misconfigured port gives: it is +// reported where it happens rather than as a server that silently never listens. +func TestWebServerReportsAPortItCannotHave(t *testing.T) { + health, err := newHealthChecker(logger.Test(t), beholder.NewNoopClient(), []services.HealthReporter{newFake()}) + require.NoError(t, err) + require.NoError(t, health.Start(t.Context())) + t.Cleanup(func() { assert.NoError(t, health.Close()) }) + checker := health.checker + + taken, err := net.Listen("tcp", ":0") + require.NoError(t, err) + t.Cleanup(func() { _ = taken.Close() }) + + // Building it registers routes and binds nothing, so the failure is at start. + port := uint16(taken.Addr().(*net.TCPAddr).Port) + web := newWebServer(logger.Test(t), HTTPConfig{Port: port}, http.NewServeMux(), checker) + + require.ErrorContains(t, web.Start(t.Context()), fmt.Sprintf("failed to listen on port %d", port)) +} diff --git a/libs/capability/pyroscope.go b/libs/capability/pyroscope.go new file mode 100644 index 000000000..e4f63074f --- /dev/null +++ b/libs/capability/pyroscope.go @@ -0,0 +1,93 @@ +package capability + +import ( + "context" + "fmt" + "runtime/debug" + + "github.com/grafana/pyroscope-go" + + commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" +) + +// PyroscopeConfig configures continuous profiling. An empty ServerAddress leaves profiling off. +type PyroscopeConfig struct { + ServerAddress string `usage:"pyroscope server address; profiling is disabled when unset"` + AuthToken commonconfig.SecretString `usage:"pyroscope auth token" flagdocs:"noexample"` + Environment string `usage:"environment tag attached to profiles"` +} + +// newProfiler returns continuous profiling as a service, or nil when no pyroscope server is +// configured. +func newProfiler(lggr logger.Logger, appName string, cfg PyroscopeConfig) *profilerService { + if cfg.ServerAddress == "" { + return nil + } + + p := &profilerService{appName: appName, cfg: cfg} + p.Service, _ = services.Config{ + Name: "Profiler", + Start: p.start, + Close: p.close, + }.NewServiceEngine(lggr) + return p +} + +func startProfiler(ctx context.Context, lggr logger.Logger, appName string, cfg PyroscopeConfig) (*profilerService, error) { + profiler := newProfiler(lggr, appName, cfg) + if profiler == nil { + return nil, nil + } + + return profiler, profiler.Start(ctx) +} + +type profilerService struct { + services.Service + + appName string + cfg PyroscopeConfig + + // profiler is what start made and close stops. Written by one hook and read by the other, which + // the state machine's lock orders: a service cannot be closed unless it started. + profiler *pyroscope.Profiler +} + +func (p *profilerService) start(context.Context) error { + var ver, sha string + if bi, ok := debug.ReadBuildInfo(); ok { + ver = bi.Main.Version + sha = bi.Main.Sum + if len(sha) > 7 { + sha = sha[:7] + } + } + + profiler, err := pyroscope.Start(pyroscope.Config{ + ApplicationName: p.appName, + ServerAddress: p.cfg.ServerAddress, + AuthToken: string(p.cfg.AuthToken), + Tags: map[string]string{ + "version": ver, + "sha": sha, + "environment": p.cfg.Environment, + }, + ProfileTypes: []pyroscope.ProfileType{ + pyroscope.ProfileCPU, + pyroscope.ProfileAllocObjects, + pyroscope.ProfileAllocSpace, + pyroscope.ProfileInuseObjects, + pyroscope.ProfileInuseSpace, + }, + }) + if err != nil { + return fmt.Errorf("failed to start profiler: %w", err) + } + + p.profiler = profiler + return nil +} + +func (p *profilerService) close() error { return p.profiler.Stop() } diff --git a/libs/capability/registry.go b/libs/capability/registry.go new file mode 100644 index 000000000..8bd0979f7 --- /dev/null +++ b/libs/capability/registry.go @@ -0,0 +1,190 @@ +package capability + +import ( + "context" + "errors" + "fmt" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + "github.com/smartcontractkit/chainlink-common/pkg/capabilities/registry" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" +) + +// newRegistry builds the registry this process holds its capabilities in and resolves others +// through, as a service. +// +// Nothing is dialled here. grpc.NewClient connects on the first RPC rather than when it is made, so +// a proxy that is not up yet delays the first lookup instead of failing the whole boot - which +// matters because the node starts this process and the two race. +// +// It does not validate the target either: an empty string and an unknown scheme both build a client +// happily, so the error below is close to unreachable and a typo shows up as a failed lookup rather +// than at startup. What catches an unset URL is the `validate:"required"` tag on the setting. +// +// It is built rather than started for the same reason telemetry is: a capability is handed the +// registry when it is constructed, which is before the root that owns this can start anything. +func newRegistry(lggr logger.Logger, cfg capabilityConfig, servers *serverFactory) (*registryService, error) { + r := ®istryService{servers: servers} + local := registry.Local(lggr) + + if cfg.ProxyURL == "" { + // No node to ask. The local registry is the whole of this process's: it resolves what this + // binary registered and nothing else, and the metadata calls - which DONs exist, what OCR + // configuration a capability runs under - fail rather than answering with something + // invented. That is what a binary run on its own wants, and it dials nothing. + r.proxy = local + } else { + conn, err := grpc.NewClient(cfg.ProxyURL, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return nil, fmt.Errorf("failed to create registry proxy client for %s: %w", cfg.ProxyURL, err) + } + r.conn = conn + + // One registry, two questions: what capabilities there are, and what configuration an OCR + // one runs under. Both are answered by whoever read the registry, over the one connection. + // + // The addresses map WithRemote takes is deliberately nil. Add on the result would announce + // the address it found written there - a convention shared between whoever serves a + // capability and whoever announces it. This service announces with AddAt instead, at the + // point the address exists, so the map would be a write nobody reads. + r.proxy = local.WithRemote(conn, nil) + } + + r.Service, r.eng = services.Config{Name: "CapabilityRegistry", Close: r.close}.NewServiceEngine(lggr) + return r, nil +} + +// startRegistry builds the registry and starts it. +func startRegistry(ctx context.Context, lggr logger.Logger, cfg capabilityConfig, servers *serverFactory) (*registryService, error) { + r, err := newRegistry(lggr, cfg, servers) + if err != nil { + return nil, err + } + return r, r.Start(ctx) +} + +// registryService is the registry this process holds its own capabilities in and resolves others +// through, the servers its own are reached on, and the connection to the node's registry - as one +// service, so that closing it undoes all three in order. +// +// There is no Start. A lazily-connected client has nothing to start, and the work with ordering in +// it - serving and announcing a capability - is Add's, called once the capability exists rather +// than when the root starts. +type registryService struct { + services.Service + eng *services.Engine + + // proxy resolves capabilities: what this binary hosts first, as the values it already holds, + // and the rest from the node behind conn. Local first is not an optimisation: a capability + // hosted here is a value this process already has, so resolving it locally hands back the + // implementation rather than a gRPC client looping back into this same process. + proxy registry.Registry + + // servers makes the gRPC server each Add serves its capability on - one per capability, since + // a registry addresses a capability by the address serving it. + servers *serverFactory + + // hosted is what Add made reachable, so close undoes exactly that, in reverse. + hosted []hosted + + // conn is the connection to the node's registry proxy, which resolutions, announcements and + // OCR questions all share. Nil when no proxy is configured, which is a binary with no node + // behind it. + conn *grpc.ClientConn +} + +// hosted is one capability being served, and the server serving it. +type hosted struct { + id string + server *server +} + +// Add makes c reachable: served on a gRPC server of its own, held in this process's registry, and +// announced to the node's. +// +// The order is the registration protocol. Serving comes first because the announcement is what +// invites traffic, so nothing is announced until it can be answered - and the address is announced +// where it is made, rather than written somewhere an Add can find it later, so the two cannot +// disagree. +// +// It is called at build time rather than from a Start of this service's own: a capability that +// cannot be announced fails the run before it is nominally up, and the health checker - which the +// run starts after this - only reports ready once this has run. The capability itself is already +// started by the caller, so traffic the announcement invites lands on something running. +func (r *registryService) Add(ctx context.Context, c Capability) error { + info, err := c.Info(ctx) + if err != nil { + return fmt.Errorf("failed to read the capability's info: %w", err) + } + + server, err := r.servers.new(ctx, logger.Named(r.eng, info.ID)) + if err != nil { + return fmt.Errorf("failed to open a server for capability %s: %w", info.ID, err) + } + // Undone here on any failure below rather than by close: a failure before the run starts this + // service means it never started, and StopOnce would refuse to run close's undo at all. + if err := registry.RegisterCapability(r.eng, server.grpcServer(), c, info.CapabilityType); err != nil { + _ = server.Close() + return fmt.Errorf("failed to serve capability %s: %w", info.ID, err) + } + if err := server.Start(ctx); err != nil { + _ = server.Close() + return fmt.Errorf("failed to start the server for capability %s: %w", info.ID, err) + } + + if err := r.proxy.Add(ctx, c); err != nil { + _ = server.Close() + return fmt.Errorf("failed to register capability %s: %w", info.ID, err) + } + + // Announced last: the announcement is what invites traffic, so nothing is announced until it + // can be served. With no node behind this process there is nothing to announce to. + if r.conn != nil { + if err := r.proxy.AddAt(ctx, info.ID, info.CapabilityType, server.address()); err != nil { + _ = r.proxy.Remove(ctx, info.ID) + _ = server.Close() + return fmt.Errorf("failed to announce capability %s: %w", info.ID, err) + } + } + + r.hosted = append(r.hosted, hosted{id: info.ID, server: server}) + r.eng.Infow("Registered capability", "capabilityID", info.ID, "type", info.CapabilityType, "address", server.address()) + return nil +} + +// close undoes every Add in reverse: stop inviting traffic (Remove drops the local hold and tells +// the node's registry to drop the address), then stop answering it, then release the connection. +// +// One ordered function rather than concurrent closes, because the steps are each other's +// preconditions: the Remove RPC has to reach the node before the connection it travels on closes. +// +// Failure to deregister is logged rather than returned. The process is going away, and a stale +// entry in a registry that cannot reach it any more is not worth failing shutdown over - the +// registry fails to dial it and drops it. +func (r *registryService) close() error { + // A context of its own rather than the engine's: Close closes the StopChan the engine's + // derives from before it runs this hook, so the deregistration would be cancelled before it + // left the process. + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + for i := len(r.hosted) - 1; i >= 0; i-- { + h := r.hosted[i] + if err := r.proxy.Remove(ctx, h.id); err != nil { + r.eng.Warnw("Failed to deregister capability", "capabilityID", h.id, "err", err) + } + r.eng.ErrorIfFn(h.server.Close, "Failed to stop the server for capability "+h.id) + } + + // The registry first: what it closes is what it dialled of its own accord - the capability + // addresses it resolved - which are connections of their own rather than this one. + err := r.proxy.Close() + if r.conn == nil { + return err + } + return errors.Join(err, r.conn.Close()) +} diff --git a/libs/capability/registry_test.go b/libs/capability/registry_test.go new file mode 100644 index 000000000..2db7d3365 --- /dev/null +++ b/libs/capability/registry_test.go @@ -0,0 +1,172 @@ +package capability + +import ( + "context" + "net" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/emptypb" + + registrypb "github.com/smartcontractkit/chainlink-common/pkg/capabilities/registry/pb" + "github.com/smartcontractkit/chainlink-common/pkg/logger" +) + +// TestNewRegistryDoesNotDial is the property the boot sequence rests on: the node starts this +// process and the two race, so a registry proxy that is not up yet has to delay the first lookup +// rather than fail the run. +// +// Nothing is listening on the port below, and building the registry still succeeds. +func TestNewRegistryDoesNotDial(t *testing.T) { + reg, err := newRegistry(logger.Test(t), capabilityConfig{ProxyURL: "localhost:1", CapabilityDonID: 1}, + &serverFactory{host: defaultHost}) + + require.NoError(t, err, "an unreachable proxy should not fail construction") + require.NotNil(t, reg.proxy) + assert.Empty(t, reg.hosted, "nothing is served yet, so nothing is announced") + + require.NoError(t, reg.Start(t.Context())) + require.NoError(t, reg.Close()) +} + +// TestNewRegistryDoesNotValidateTheTarget pins something worth knowing before debugging one: not +// even a nonsense target fails here. +// +// grpc.NewClient defers everything to the first RPC, so a typo in --capabilities.proxy-url is not +// reported at startup - it surfaces as a failed capability lookup later on. What catches an unset +// one is the `validate:"required"` tag on the setting, not this. +func TestNewRegistryDoesNotValidateTheTarget(t *testing.T) { + for _, target := range []string{"", "!://not a target", "unknownscheme:///x"} { + t.Run(target, func(t *testing.T) { + reg, err := newRegistry(logger.Test(t), capabilityConfig{ProxyURL: target}, &serverFactory{host: defaultHost}) + require.NoError(t, err) + require.NoError(t, reg.Start(t.Context())) + assert.NoError(t, reg.Close()) + }) + } +} + +// TestNewRegistryWithoutAProxyIsLocalOnly covers a binary with no node behind it: it dials nothing, +// and resolves only what it hosts. +// +// The metadata calls failing is the point rather than a shortcoming. A process holding capability +// values has no way to know which DONs exist, so saying so beats answering with something invented. +func TestNewRegistryWithoutAProxyIsLocalOnly(t *testing.T) { + reg, err := newRegistry(logger.Test(t), capabilityConfig{CapabilityDonID: 1}, &serverFactory{host: defaultHost}) + require.NoError(t, err) + t.Cleanup(func() { assert.NoError(t, reg.Close()) }) + require.NoError(t, reg.Start(t.Context())) + + assert.Nil(t, reg.conn, "nothing should have been dialled") + require.NotNil(t, reg.proxy) + + _, err = reg.proxy.DONByID(t.Context(), 1) + assert.Error(t, err, "there is no node to ask, so the metadata calls should fail") +} + +// TestRegistryAddServesAndHolds covers the local half of being reachable: the capability is served +// on an address of its own, and this process's registry resolves it. +func TestRegistryAddServesAndHolds(t *testing.T) { + reg, err := newRegistry(logger.Test(t), capabilityConfig{}, &serverFactory{host: defaultHost}) + require.NoError(t, err) + require.NoError(t, reg.Start(t.Context())) + + require.NoError(t, reg.Add(t.Context(), newFake())) + + // Served: the hosted server's address answers. + require.Len(t, reg.hosted, 1) + conn, err := net.DialTimeout("tcp", reg.hosted[0].server.address(), 5*time.Second) + require.NoError(t, err) + _ = conn.Close() + + // Held: the registry resolves the capability as a value. + got, err := reg.proxy.Get(t.Context(), fakeID) + require.NoError(t, err) + assert.NotNil(t, got) + + // Close takes both back. Remove hollows the registry's entry out rather than deleting it, so + // what the ID still maps to answers nothing. + require.NoError(t, reg.Close()) + + got, err = reg.proxy.Get(t.Context(), fakeID) + if err == nil { + _, err = got.Info(t.Context()) + } + require.Error(t, err, "the registry should no longer hold the capability") +} + +// TestRegistryAddAnnouncesToTheNode covers the remote half: with a proxy configured, adding the +// capability announces the address it is served at, and closing takes the announcement back before +// the connection it travelled on closes. +func TestRegistryAddAnnouncesToTheNode(t *testing.T) { + stub := &stubRegistry{adds: map[string]string{}} + + reg, err := newRegistry(logger.Test(t), capabilityConfig{ProxyURL: serveStubRegistry(t, stub)}, + &serverFactory{host: defaultHost}) + require.NoError(t, err) + require.NoError(t, reg.Start(t.Context())) + + require.NoError(t, reg.Add(t.Context(), newFake())) + + // Announced at the address it is served at, and that address answers. + require.Len(t, reg.hosted, 1) + addr, ok := stub.announced(fakeID) + require.True(t, ok, "the node's registry should know the capability") + assert.Equal(t, reg.hosted[0].server.address(), addr) + conn, err := net.DialTimeout("tcp", addr, 5*time.Second) + require.NoError(t, err) + _ = conn.Close() + + require.NoError(t, reg.Close()) + stub.mu.Lock() + defer stub.mu.Unlock() + assert.Contains(t, stub.removes, fakeID, "shutdown should take the announcement back") +} + +// stubRegistry is the node's registry, minimally: it records what is announced to it, and what is +// taken back. +type stubRegistry struct { + registrypb.UnimplementedCapabilitiesRegistryServer + + mu sync.Mutex + adds map[string]string // capability ID -> the address it was announced at + removes []string +} + +func (s *stubRegistry) Add(_ context.Context, req *registrypb.AddRequest) (*emptypb.Empty, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.adds[req.CapabilityId] = req.CallbackUrl + return &emptypb.Empty{}, nil +} + +func (s *stubRegistry) Remove(_ context.Context, req *registrypb.RemoveRequest) (*emptypb.Empty, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.removes = append(s.removes, req.CapabilityId) + return &emptypb.Empty{}, nil +} + +func (s *stubRegistry) announced(id string) (string, bool) { + s.mu.Lock() + defer s.mu.Unlock() + addr, ok := s.adds[id] + return addr, ok +} + +// serveStubRegistry runs a stub registry on a free port and returns its address. +func serveStubRegistry(t *testing.T, stub *stubRegistry) string { + t.Helper() + + listener, err := net.Listen("tcp", "localhost:0") + require.NoError(t, err) + gsrv := grpc.NewServer() + registrypb.RegisterCapabilitiesRegistryServer(gsrv, stub) + go func() { _ = gsrv.Serve(listener) }() + t.Cleanup(gsrv.Stop) + return listener.Addr().String() +} diff --git a/libs/capability/run.go b/libs/capability/run.go new file mode 100644 index 000000000..f89f52fd3 --- /dev/null +++ b/libs/capability/run.go @@ -0,0 +1,225 @@ +package capability + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "os/signal" + "path/filepath" + "reflect" + "syscall" + + "github.com/spf13/cobra" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + + "github.com/smartcontractkit/chainlink-common/pkg/config/flags" + "github.com/smartcontractkit/chainlink-common/pkg/logger" +) + +// Run builds and runs the capability constructor makes, and does not return until ctx is cancelled or the +// process is signalled. +// +// func main() { capability.Run(context.Background(), trigger.NewCron) } +func Run(ctx context.Context, constructor any, opts ...Option) { + lggr, err := NewLogger() + if err != nil { + log.Fatal(err) + } + + if err := RunErr(ctx, lggr, constructor, opts...); err != nil { + lggr.Fatal(err) + } +} + +// RunErr is Run, returning the error rather than exiting on it. +// +// A run blocks until it is told to stop, which is either ctx being cancelled or this process being +// signalled - see below. +func RunErr(ctx context.Context, lggr logger.Logger, constructor any, opts ...Option) error { + if lggr == nil { + return errors.New("must provide a logger") + } + + // We instantiate this up-front so we can bubble up an error message about an invalid `constructor` early. + c, err := newConstructor(constructor) + if err != nil { + return err + } + + // TODO: Run should accept some kind of Info struct + root := &cobra.Command{ + Use: filepath.Base(os.Args[0]), + Short: "A CRE capability", + } + root.PersistentFlags().String("config", "", "Path to config file") + + lggr = logger.Named(lggr, root.Name()) + + cfg := defaultConfig() + for _, opt := range opts { + opt(cfg) + } + if err := cfg.bind(root); err != nil { + return err + } + + // Bind the capability's config if the constructor function declares a config struct. + // This will be namespaced using the name of the root command. + // eg. --cron.fastest-schedule-interval-seconds + if err := c.bindCapabilityConfig(root); err != nil { + return err + } + + root.AddCommand(&cobra.Command{ + Use: "run", + Short: "Run the capability", + RunE: func(cmd *cobra.Command, _ []string) error { + // TODO: pass in constructors here for our runnable/embeddable dependencies? + return run(cmd.Context(), lggr, cmd.Root().Name(), cfg, c) + }, + }) + + // TODO: embed command + root.AddCommand(&cobra.Command{ + Use: "embed", + Short: "Embed the capability", + RunE: func(cmd *cobra.Command, _ []string) error { + // TODO: pass in constructors here for our runnable/embeddable dependencies? + // Something else here + return run(cmd.Context(), lggr, cmd.Root().Name(), cfg, c) + }, + }) + + // Wire up SIGTERM + SIGINT to the context + ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM) + defer stop() + + return root.ExecuteContext(ctx) +} + +type constructor struct { + fn reflect.Value + typ reflect.Type + + // configIn is the index of the parameter that is the capability's own config, or -1 when the + // constructor declares none. + configIn int + // config is the instance of that parameter the flags decode fills. Kept here rather than + // passed to run because the decode hook fills it in place when the command runs: call has to + // read that same memory, not a copy made when it was bound. Valid only when configIn >= 0. + config reflect.Value +} + +var capabilityType = reflect.TypeFor[Capability]() + +// newConstructor validates that constructor has the right shape, i.e.: +// - it's a function +// - and the function returns a capability and optionally an error +func newConstructor(ctor any) (constructor, error) { + t := reflect.TypeOf(ctor) + if t == nil || t.Kind() != reflect.Func { + return constructor{}, fmt.Errorf("a capability constructor must be a function, got %T", ctor) + } + + switch t.NumOut() { + case 1: + case 2: + if t.Out(1) != reflect.TypeFor[error]() { + return constructor{}, fmt.Errorf( + "a capability constructor returning two values must return an error second, got %s", t) + } + default: + return constructor{}, fmt.Errorf( + "a capability constructor must return the capability, and optionally an error, got %s", t) + } + + if out := t.Out(0); !out.Implements(capabilityType) { + return constructor{}, fmt.Errorf("%s does not implement capability.Capability", out) + } + + configIn := -1 + for i := range t.NumIn() { + if !isConfig(t.In(i)) { + continue + } + if configIn >= 0 { + return constructor{}, fmt.Errorf("a capability constructor takes its config once, got %s", t) + } + configIn = i + } + return constructor{fn: reflect.ValueOf(ctor), typ: t, configIn: configIn}, nil +} + +var isConfigType = reflect.TypeFor[Config]() + +// isConfig reports whether t declares itself a capability's config by embedding Config. The +// embedding has to be direct: the marker declares the struct it is embedded in, not the structs +// that embed that one. +func isConfig(t reflect.Type) bool { + if t.Kind() != reflect.Struct { + return false + } + for i := range t.NumField() { + if f := t.Field(i); f.Anonymous && f.Type == isConfigType { + return true + } + } + return false +} + +// bindCapabilityConfig creates the config the constructor declares, if any, and binds its fields as flags +// on root, under root's name. +func (c *constructor) bindCapabilityConfig(root *cobra.Command) error { + if c.configIn < 0 { + return nil + } + + v := reflect.New(c.typ.In(c.configIn)) + fopts := flags.DefaultTOMLOptions("CRE", "CL") + fopts.Namespace = root.Name() + if err := flags.RegisterCommandFlags(root, v.Interface(), fopts); err != nil { + return fmt.Errorf("failed to register the capability's config: %w", err) + } + c.config = v.Elem() + return nil +} + +// call builds the capability by calling the contstructor. +// The constructor's parameters are scanned, and type matched against the dependency set. +// The config parameter is treated exceptionally and hydrated from the configuration that was passed in. +func (c constructor) call(deps Dependencies) (Capability, error) { + args := make([]reflect.Value, c.typ.NumIn()) + for i := range args { + if i == c.configIn { + args[i] = c.config + continue + } + + want := c.typ.In(i) + + v, ok := deps.resolve(want) + if !ok { + return nil, fmt.Errorf("the capability constructor asks for a %s, and nothing in this run "+ + "provides one: %s", want, c.typ) + } + args[i] = v + } + + out := c.fn.Call(args) + if len(out) == 2 && !out[1].IsNil() { + return nil, fmt.Errorf("failed to build the capability: %w", out[1].Interface().(error)) + } + return out[0].Interface().(Capability), nil +} + +type Option func(*config) + +// WithOtelViews sets otel metric views - histogram bucket boundaries, typically - on the beholder +// client this process reports through. +// +// capability.Run(ctx, trigger.NewCron, capability.WithOtelViews(trigger.MetricViews()...)) +func WithOtelViews(views ...sdkmetric.View) Option { + return func(c *config) { c.observability.otelViews = append(c.observability.otelViews, views...) } +} diff --git a/libs/capability/run_test.go b/libs/capability/run_test.go new file mode 100644 index 000000000..64b5245ea --- /dev/null +++ b/libs/capability/run_test.go @@ -0,0 +1,386 @@ +package capability + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "os" + "slices" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/reflect/protoreflect" + + "github.com/smartcontractkit/chainlink-common/pkg/capabilities" + capabilitiespb "github.com/smartcontractkit/chainlink-common/pkg/capabilities/pb" + "github.com/smartcontractkit/chainlink-common/pkg/capabilities/registry" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + "github.com/smartcontractkit/chainlink-common/pkg/types/core" +) + +// fake stands in for a generated capability server, which is what a real constructor returns. Only +// its type matters here: nothing calls it, because building it is as far as this step goes. +// +// The interfaces are embedded rather than implemented so that this says what a capability is - a +// service, a capability, and the proto service behind it - without a page of methods saying it. +type fake struct { + runnable + capabilities.ExecutableAndTriggerCapability + + // info is what the fake reports itself as, which is what the registrar serves and announces + // it by. + info capabilities.CapabilityInfo + + // started and closed record what the root service did with it. Written on the run's goroutine + // and read on the test's, but only after the run has returned, so the channel that reports that + // is what orders them. + started bool + closed bool +} + +// runnable holds the services.Service half. It is a type of its own so that the field it embeds is +// not called Service, which would collide with the method below. +type runnable struct{ services.Service } + +// Service is the base capability's descriptor rather than nil: the debug UI builds its page from +// what this returns and refuses a capability without one. +func (fake) Service() protoreflect.ServiceDescriptor { + return capabilitiespb.File_capabilities_proto.Services().ByName("BaseCapability") +} + +func (f *fake) Info(context.Context) (capabilities.CapabilityInfo, error) { return f.info, nil } + +// fakeID is what the fake registers and announces itself as. +const fakeID = "fake@1.0.0" + +// newFake builds one with a real service behind it, since the health checker asks the capability +// its name and whether it is ready. +func newFake() *fake { + f := &fake{} + f.info, _ = capabilities.NewCapabilityInfo(fakeID, capabilities.CapabilityTypeCombined, "a fake") + f.runnable.Service, _ = services.Config{ + Name: "FakeCapability", + Start: func(context.Context) error { f.started = true; return nil }, + Close: func() error { f.closed = true; return nil }, + }.NewServiceEngine(logger.Nop()) + return f +} + +// execute drives the real entry point with args of its own, and stops it once it is up. +// +// os.Args rather than a seam into the command tree, because that is what RunErr reads and what a +// binary is actually started with - including argv[0], which is what names the root command. +// +// A run blocks until its context is cancelled, so this waits until the binary is serving and then +// cancels - which is a test standing in for the signal an operator would send. A run that fails on +// the way up never gets there and reports that instead. +func execute(t *testing.T, lggr logger.Logger, ctor any, args ...string) error { + t.Helper() + + // A free port rather than the default, since tests share a machine and run alongside each + // other - two of them on 8080 would collide. It is also how this tells that the binary is up. + port := 0 + if slices.Contains(args, "run") && !slices.Contains(args, "--http.port") { + port = freePort(t) + args = append(args, "--http.port", strconv.Itoa(port)) + + // No --capabilities.proxy-url: it is optional, and leaving it out is the simpler run - the + // registry then resolves only what this binary hosts, and dials nothing. + } + + previous := os.Args + t.Cleanup(func() { os.Args = previous }) + os.Args = append([]string{"cron"}, args...) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + done := make(chan error, 1) + go func() { done <- RunErr(ctx, lggr, ctor) }() + + if port == 0 { + // Nothing that serves: a rejected constructor, or a bare invocation that prints help. + return <-done + } + + deadline := time.After(30 * time.Second) + for { + select { + case err := <-done: + return err + case <-deadline: + cancel() + t.Fatal("the run never started serving") + return nil + case <-time.After(5 * time.Millisecond): + if serving(port) { + cancel() + return <-done + } + } + } +} + +// serving reports whether the shared HTTP server is answering on port, which is how a test tells +// that a run has finished coming up. +// +// A timeout, because the port is not always the run's: a test that takes the port to watch the +// run fail leaves a listener that accepts and never answers, and a bare Get would hang on it +// rather than report not-serving. +func serving(port int) bool { + client := &http.Client{Timeout: 5 * time.Second} + res, err := client.Get(fmt.Sprintf("http://localhost:%d/metrics", port)) + if err != nil { + return false + } + _ = res.Body.Close() + return res.StatusCode == http.StatusOK +} + +// freePort is a port nothing is listening on, found by listening on one and stopping again. +func freePort(t *testing.T) int { + t.Helper() + + l, err := net.Listen("tcp", ":0") + require.NoError(t, err) + port := l.Addr().(*net.TCPAddr).Port + require.NoError(t, l.Close()) + return port +} + +func TestRunErrRejects(t *testing.T) { + lggr := logger.Test(t) + + t.Run("no logger", func(t *testing.T) { + require.ErrorContains(t, RunErr(context.Background(), nil, newFake), "must provide a logger") + }) + + t.Run("something that is not a function", func(t *testing.T) { + require.ErrorContains(t, RunErr(context.Background(), lggr, "cron"), "must be a function, got string") + }) + + t.Run("nothing at all", func(t *testing.T) { + require.ErrorContains(t, RunErr(context.Background(), lggr, nil), "must be a function") + }) + + t.Run("a function returning something that cannot be hosted", func(t *testing.T) { + require.ErrorContains(t, RunErr(context.Background(), lggr, func() string { return "" }), + "string does not implement capability.Capability") + }) + + t.Run("a function returning nothing", func(t *testing.T) { + require.ErrorContains(t, RunErr(context.Background(), lggr, func() {}), "must return the capability") + }) + + t.Run("a function whose second result is not an error", func(t *testing.T) { + require.ErrorContains(t, RunErr(context.Background(), lggr, func() (*fake, string) { return nil, "" }), + "must return an error second") + }) +} + +func TestRunBuildsTheCapability(t *testing.T) { + built := 0 + + require.NoError(t, execute(t, logger.Test(t), func() *fake { built++; return newFake() }, "run")) + assert.Equal(t, 1, built, "the constructor should have been called exactly once") +} + +func TestRunReportsAFailedConstructor(t *testing.T) { + err := execute(t, logger.Test(t), func() (*fake, error) { return nil, errors.New("no schedule") }, "run") + require.ErrorContains(t, err, "failed to build the capability: no schedule") +} + +// TestRunNamesWhatItCannotProvide covers a parameter nothing answers: it is reported before +// anything starts, and names the type that went unmatched rather than the position it was in. +func TestRunNamesWhatItCannotProvide(t *testing.T) { + err := execute(t, logger.Test(t), func(string) *fake { return newFake() }, "run") + require.ErrorContains(t, err, "asks for a string, and nothing in this run provides one") +} + +// TestRunPassesTheLoggerToTheConstructor covers the first dependency: this process's logger, so a +// capability names its own from it rather than building one the operator cannot configure. +func TestRunPassesTheLoggerToTheConstructor(t *testing.T) { + var got logger.Logger + + require.NoError(t, execute(t, logger.Test(t), func(l logger.Logger) *fake { + got = l + return newFake() + }, "run")) + + assert.NotNil(t, got, "the constructor should have been handed the run's logger") +} + +// TestRunPassesTheRegistryToTheConstructor covers the registry a capability resolves other +// capabilities through. +// +// By type rather than by position, so a constructor asks for what it needs and takes nothing it +// does not - which is what lets the one below coexist with the no-argument constructors elsewhere +// in these tests. +func TestRunPassesTheRegistryToTheConstructor(t *testing.T) { + var got registry.Registry + + require.NoError(t, execute(t, logger.Test(t), func(r registry.Registry) *fake { + got = r + return newFake() + }, "run")) + + assert.NotNil(t, got, "the constructor should have been handed the run's registry") +} + +// TestRunPassesTheLimitsFactoryToTheConstructor covers the second dependency: what a capability +// bounds a workflow's requests with. +// +// A struct rather than an interface, which the type matching handles the same way - what a +// constructor asks for is a type, not a kind of type. +func TestRunPassesTheLimitsFactoryToTheConstructor(t *testing.T) { + var got limits.Factory + + require.NoError(t, execute(t, logger.Test(t), func(f limits.Factory) *fake { + got = f + return newFake() + }, "run")) + + assert.NotNil(t, got.Settings, "the factory should be built over this run's settings") + assert.NotNil(t, got.Meter) +} + +// TestRunPassesEveryDependencyItHas covers a constructor asking for more than one, in an order of +// its own: matching is by type, so the order it lists them in is its business. +func TestRunPassesEveryDependencyItHas(t *testing.T) { + var ( + gotLimits limits.Factory + gotRegistry core.CapabilitiesRegistry + ) + + require.NoError(t, execute(t, logger.Test(t), func(f limits.Factory, r core.CapabilitiesRegistry) *fake { + gotLimits, gotRegistry = f, r + return newFake() + }, "run")) + + assert.NotNil(t, gotLimits.Settings) + assert.NotNil(t, gotRegistry) +} + +// testConfig is a capability's own config, declared by embedding Config: the one parameter an +// operator supplies rather than the run resolves. +type testConfig struct { + Config + Shout string `usage:"what the fake says"` +} + +// TestRunHandsTheConstructorItsConfig covers the parameter that is not a dependency: the +// capability's own settings, bound under the binary's name and handed over decoded. The binary is +// named cron - execute's argv[0] - so its flag is --cron.shout. +func TestRunHandsTheConstructorItsConfig(t *testing.T) { + var got testConfig + + require.NoError(t, execute(t, logger.Test(t), func(cfg testConfig) *fake { + got = cfg + return newFake() + }, "run", "--cron.shout", "hello")) + + assert.Equal(t, "hello", got.Shout) +} + +// TestRunReadsTheCapabilityConfigFromTheEnvironment is the same binding from the other direction: +// CRE_, then the binary's name, as every other section's settings are. +func TestRunReadsTheCapabilityConfigFromTheEnvironment(t *testing.T) { + t.Setenv("CRE_CRON_SHOUT", "from the environment") + + var got testConfig + + require.NoError(t, execute(t, logger.Test(t), func(cfg testConfig) *fake { + got = cfg + return newFake() + }, "run")) + + assert.Equal(t, "from the environment", got.Shout) +} + +// TestRunHandsTheConstructorItsConfigAndItsDependencies covers the two kinds of parameter side by +// side: the config from the flags, the dependencies from the run, and neither answered by the +// other. +func TestRunHandsTheConstructorItsConfigAndItsDependencies(t *testing.T) { + var ( + gotCfg testConfig + gotLimits limits.Factory + ) + + require.NoError(t, execute(t, logger.Test(t), func(cfg testConfig, f limits.Factory) *fake { + gotCfg, gotLimits = cfg, f + return newFake() + }, "run", "--cron.shout", "hello")) + + assert.Equal(t, "hello", gotCfg.Shout) + assert.NotNil(t, gotLimits.Settings) +} + +// TestRunRejectsTwoConfigs pins the one-config rule: the struct is bound under the binary's name, +// so two of them would share one namespace and which setting belonged to which would be anyone's +// guess. +func TestRunRejectsTwoConfigs(t *testing.T) { + err := execute(t, logger.Test(t), func(a, b testConfig) *fake { return newFake() }, "run") + require.ErrorContains(t, err, "takes its config once") +} + +// TestRunRejectsTheDependenciesStruct pins the rule the other way round: a constructor declares the +// things it uses, not the bag they came in. +// +// Taking the struct would be a dependency on everything a run has, and adding a field to it would +// silently widen what every capability appeared to need. +func TestRunRejectsTheDependenciesStruct(t *testing.T) { + err := execute(t, logger.Test(t), func(Dependencies) *fake { return newFake() }, "run") + + require.ErrorContains(t, err, "asks for a capability.Dependencies, and nothing in this run provides one") +} + +// TestRunPassesTheRegistryAsANarrowerInterface covers matching by assignability: a capability that +// only needs to resolve capabilities can say so, without naming the wider type the run holds. +func TestRunPassesTheRegistryAsANarrowerInterface(t *testing.T) { + var got core.CapabilitiesRegistry + + require.NoError(t, execute(t, logger.Test(t), func(r core.CapabilitiesRegistry) *fake { + got = r + return newFake() + }, "run")) + + assert.NotNil(t, got) +} + +// TestRunHasARunCommand is the shape of the binary rather than what it does: a root that runs +// nothing itself, and a way to start it hanging off it. +func TestRunHasARunCommand(t *testing.T) { + // A bare invocation prints help and does nothing, rather than starting the capability. + require.NoError(t, execute(t, logger.Test(t), func() *fake { + t.Error("the root command should not build the capability") + return newFake() + })) +} + +// TestRunServesAndAnnouncesTheCapability is the whole path through the real entry point: a run +// serves its capability, tells the node's registry where, and takes it back on shutdown. +// +// The removal half is reliable to assert here because the registry's close is one ordered +// function: the Remove RPC is sent before the connection it travels on is closed. +func TestRunServesAndAnnouncesTheCapability(t *testing.T) { + stub := &stubRegistry{adds: map[string]string{}} + proxyURL := serveStubRegistry(t, stub) + + require.NoError(t, execute(t, logger.Test(t), func() *fake { return newFake() }, + "run", "--capabilities.proxy-url", proxyURL)) + + addr, ok := stub.announced(fakeID) + require.True(t, ok, "the run should have announced the capability to the node's registry") + assert.NotEmpty(t, addr) + + stub.mu.Lock() + defer stub.mu.Unlock() + assert.Contains(t, stub.removes, fakeID, "and deregistered it on the way out") +} diff --git a/libs/capability/runner.go b/libs/capability/runner.go new file mode 100644 index 000000000..81523682d --- /dev/null +++ b/libs/capability/runner.go @@ -0,0 +1,139 @@ +package capability + +import ( + "context" + "fmt" + "net/http" + "os" + + "github.com/hashicorp/go-plugin" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/loop" + "github.com/smartcontractkit/chainlink-common/pkg/services" +) + +func run(ctx context.Context, lggr logger.Logger, name string, cfg *config, newCapability constructor) error { + defer func() { _ = lggr.Sync() }() + + var svcs []services.Service + defer func() { + if err := services.MultiCloser(svcs).Close(); err != nil { + logger.Sugared(lggr).Errorw("failed to stop the services of this run", "err", err) + } + }() + + mux := http.NewServeMux() + + profiler, err := startProfiler(ctx, lggr, name, cfg.observability.pyroscope) + if err != nil { + return fmt.Errorf("failed to start profiler: %w", err) + } + if profiler != nil { + svcs = append(svcs, profiler) + } + + telemetry, err := startTelemetry(ctx, lggr, &cfg.observability) + if err != nil { + return fmt.Errorf("failed to build telemetry: %w", err) + } + svcs = append(svcs, telemetry) + + reg, err := startRegistry(ctx, lggr, cfg.capabilities, &serverFactory{ + host: cfg.grpc.AdvertiseHost, + startPort: cfg.grpc.StartPort, + }) + if err != nil { + return fmt.Errorf("failed to start registry: %w", err) + } + svcs = append(svcs, reg) + + settings, err := newSettings(lggr) + if err != nil { + return err + } + mux.HandleFunc(reloadPath(), reloadHandler(lggr, settings, settingsPath())) + + capability, err := startCapability(ctx, lggr, cfg, newCapability, reg, settings, mux) + if err != nil { + return err + } + svcs = append(svcs, capability) + + health, err := startHealthChecker(ctx, lggr, beholder.GetClient(), servicesToHealthReporters(svcs)) + if err != nil { + return fmt.Errorf("failed to start health checker: %w", err) + } + svcs = append(svcs, health) + + ws, err := startWebServer(ctx, lggr, cfg.observability.http, mux, health.checker) + if err != nil { + return fmt.Errorf("failed to start web server: %w", err) + } + svcs = append(svcs, ws) + + if underPluginHost() { + lggr.Info("Serving the empty LOOP: this process is supervised by a go-plugin host") + plugin.Serve(&plugin.ServeConfig{ + HandshakeConfig: loop.EmptyHandshakeConfig(), + Plugins: map[string]plugin.Plugin{loop.PluginEmptyName: &loop.EmptyLoop{}}, + GRPCServer: plugin.DefaultGRPCServer, + }) + return nil + } + + <-ctx.Done() + lggr.Info("Shutting down") + return nil +} + +func servicesToHealthReporters(svcs []services.Service) []services.HealthReporter { + reporters := make([]services.HealthReporter, 0, len(svcs)) + for _, s := range svcs { + reporters = append(reporters, s) + } + return reporters +} + +// underPluginHost reports whether this process was launched by a go-plugin host, detected via the +// empty plugin's handshake magic cookie. +// +// The check is necessary rather than defensive: go-plugin's Serve refuses to run - and exits the +// process - when the cookie is absent, so a standalone binary that called it would die on startup. +func underPluginHost() bool { + h := loop.EmptyHandshakeConfig() + return os.Getenv(h.MagicCookieKey) == h.MagicCookieValue +} + +// startCapability builds the capability from its constructor and makes it reachable: started, +// served and announced by the registry, and mounted via the debug UI if debug mode is turned on. +func startCapability(ctx context.Context, lggr logger.Logger, cfg *config, ctor constructor, reg *registryService, settings *loop.AtomicSettings, mux *http.ServeMux) (Capability, error) { + c, err := ctor.call(Dependencies{ + Logger: lggr, + CapabilityRegistry: reg.proxy, + LimitsFactory: newLimitsFactory(lggr, settings), + }) + if err != nil { + return nil, fmt.Errorf("failed to instantiate capability: %w", err) + } + + if err := c.Start(ctx); err != nil { + return nil, fmt.Errorf("failed to start capability: %w", err) + } + + if err := reg.Add(ctx, c); err != nil { + // Not yet on the caller's list, so the deferred close would never reach it. + _ = c.Close() + return nil, err + } + + if cfg.capabilities.HTTPDebug { + if err := mountDebugUI(ctx, lggr, mux, reg.proxy, c); err != nil { + _ = c.Close() + return nil, err + } + } + + return c, nil +} diff --git a/libs/capability/runner_test.go b/libs/capability/runner_test.go new file mode 100644 index 000000000..4a2841ada --- /dev/null +++ b/libs/capability/runner_test.go @@ -0,0 +1,280 @@ +package capability + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/loop" +) + +// runArgs is the arguments of one run over settings a test can start: a real port, and everything +// else off. +func runArgs(t *testing.T, ctor any) (*config, constructor, uint16) { + t.Helper() + + c, err := newConstructor(ctor) + require.NoError(t, err) + + port := uint16(freePort(t)) + cfg := defaultConfig() + cfg.observability.http.Port = port + // A stub the registrar's announcement can land on: registering the capability dials the + // proxy, so it has to be a real listener rather than merely a lazy one. + cfg.capabilities.ProxyURL = serveStubRegistry(t, &stubRegistry{adds: map[string]string{}}) + + return cfg, c, port +} + +// runToCompletion starts a run and stops it once it is serving, returning what run returned. +// +// A run blocks until its context is cancelled, so a test that wants one to finish has to be the +// thing that ends it - standing in for the signal an operator would send. A run that fails on the +// way up never gets as far as serving and reports that instead. +func runToCompletion(t *testing.T, cfg *config, c constructor, port uint16) error { + t.Helper() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + done := make(chan error, 1) + go func() { done <- run(ctx, logger.Test(t), "cron", cfg, c) }() + + deadline := time.After(30 * time.Second) + for { + select { + case err := <-done: + return err + case <-deadline: + cancel() + t.Fatal("the run never started serving") + return nil + case <-time.After(5 * time.Millisecond): + if serving(int(port)) { + cancel() + return <-done + } + } + } +} + +// TestRunRunsAndUnwinds is run's whole contract in one: it brings the process up, and a run that has +// finished has put back everything it changed. +// +// The port is what makes the unwind visible. A web server that was serving and is not any more is +// one that gave the port back, which nothing else in the run reports. +func TestRunRunsAndUnwinds(t *testing.T) { + built := 0 + cfg, c, port := runArgs(t, func() *fake { built++; return newFake() }) + + require.NoError(t, runToCompletion(t, cfg, c, port)) + assert.Equal(t, 1, built) + + assertPortFree(t, port) +} + +// TestRunStartsTheCapability is what the root service buys: the capability is not merely built, it +// is running - which is what makes a health check mean anything. +func TestRunStartsTheCapability(t *testing.T) { + f := newFake() + cfg, c, port := runArgs(t, func() *fake { return f }) + + require.NoError(t, runToCompletion(t, cfg, c, port)) + + assert.True(t, f.started, "the run should have started the capability") + assert.True(t, f.closed, "and closed it again on the way out") +} + +// TestRunInstallsTelemetryBeforeBuildingTheCapability is the reason telemetry is started where it +// is, rather than with everything else the run supervises. +// +// A capability creates its instruments while it is being constructed - cron does, in NewMetrics - +// and an OTEL instrument resolves beholder.GetMeter() once, at creation. A capability built before +// this client became the process's would hold a noop meter for the life of the process, recording +// nothing, with no error anywhere to say so. +// +// Slow, and unavoidably so: it is the only test that builds a real beholder client, and closing one +// flushes to a collector that is not there - about 20s of export timeouts. The timeouts are on +// beholder.Config, which beholderConfig builds from the settings, so shortening them would mean a +// knob in production code that exists only for this. +func TestRunInstallsTelemetryBeforeBuildingTheCapability(t *testing.T) { + if testing.Short() { + t.Skip("builds a real beholder client; ~20s of export timeouts on close") + } + + cfg, _, port := runArgs(t, newFake) + // Nothing is listening there, which is fine: the client is built and installed without dialling. + cfg.observability.telemetry.Endpoint = fmt.Sprintf("localhost:%d", freePort(t)) + + var duringBuild *beholder.Client + c, err := newConstructor(func() *fake { + duringBuild = beholder.GetClient() + return newFake() + }) + require.NoError(t, err) + + before := beholder.GetClient() + require.NoError(t, runToCompletion(t, cfg, c, port)) + + require.NotNil(t, duringBuild, "the constructor should have run") + assert.NotSame(t, before, duringBuild, "the capability was built before telemetry was installed") + assert.Same(t, before, beholder.GetClient(), "and the previous client should be back afterwards") +} + +// TestRunUnwindsWhatStartedBeforeAFailure covers a constructor that fails after the observability +// services have been built: the run reports it and leaves nothing of itself behind. +// +// The beholder assertion below holds trivially here and is not evidence about telemetry. These +// settings configure no endpoint, so newTelemetry returns the noop service and installs nothing - +// there is nothing to put back. A run with telemetry configured would fail this if it asserted +// anything, which is the gap documented on the telemetry block in run: the global is swapped when +// telemetry is built, and the undo only works once the root has started it. +func TestRunUnwindsWhatStartedBeforeAFailure(t *testing.T) { + before := beholder.GetClient() + cfg, c, port := runArgs(t, func() (*fake, error) { return nil, errors.New("no schedule") }) + require.Empty(t, cfg.observability.telemetry.Endpoint, "these settings leave telemetry off") + + require.ErrorContains(t, runToCompletion(t, cfg, c, port), "no schedule") + + assert.Same(t, before, beholder.GetClient(), "nothing was installed, so nothing changed") + // The web server never got as far as starting, so the port was never taken. + assertPortFree(t, port) +} + +// TestRunReportsWhereItFailed pins that a failure names the service that would not start, rather +// than arriving as something further in. +// +// It is also what proves run gets as far as the web server on the configured port at all. +func TestRunReportsWhereItFailed(t *testing.T) { + cfg, c, port := runArgs(t, newFake) + + // Something else already has the port the web server is configured for. + taken, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + require.NoError(t, err) + t.Cleanup(func() { _ = taken.Close() }) + + require.ErrorContains(t, runToCompletion(t, cfg, c, port), + fmt.Sprintf("failed to listen on port %d", port)) +} + +// TestRunStaysUpUntilItIsToldToStop is the whole point of the wait: a run keeps serving until +// something ends it, and only then unwinds. +// +// It is also the only test that looks at a run from outside while it is up: everything is still +// serving when the assertions below are made. +func TestRunStaysUpUntilItIsToldToStop(t *testing.T) { + cfg, c, port := runArgs(t, newFake) + ctx, cancel := context.WithCancel(context.Background()) + + done := make(chan error, 1) + go func() { done <- run(ctx, logger.Test(t), "cron", cfg, c) }() + + require.Eventually(t, func() bool { return serving(int(port)) }, 30*time.Second, 5*time.Millisecond, + "the run should be serving while it is up") + + // The root is started, and the health checker reports on the root - so a run that is up is a + // run that says it is ready. This is what starting the capability bought. + for path, want := range map[string]int{"/healthz": 200, "/readyz": 200} { + res, err := http.Get(fmt.Sprintf("http://localhost:%d%s", port, path)) + require.NoError(t, err, path) + require.NoError(t, res.Body.Close()) + assert.Equal(t, want, res.StatusCode, path) + } + + // Still up: it has not unwound just because it finished starting. + select { + case err := <-done: + t.Fatalf("the run ended on its own: %v", err) + case <-time.After(50 * time.Millisecond): + } + + cancel() + select { + case err := <-done: + // Being asked to stop is not a failure: a binary that exited because it was told to should + // exit 0. + require.NoError(t, err) + case <-time.After(30 * time.Second): + t.Fatal("the run did not stop when it was told to") + } + + assertPortFree(t, port) +} + +// TestRunServesTheSettingsReloadEndpoint is the endpoint through the real run: the node dumps new +// settings to the file and hits /reload/settings.txt, and a 200 says every limit in the process now +// resolves against them. +// +// The handler reads the shared settings path - that path is the contract with the node - so this +// writes the real file and removes it again after: a leftover would be read by every other run in +// this package. +func TestRunServesTheSettingsReloadEndpoint(t *testing.T) { + require.NoError(t, os.MkdirAll(filepath.Dir(settingsPath()), 0o700)) + require.NoError(t, os.WriteFile(settingsPath(), []byte("[global]\n"), 0o600)) + t.Cleanup(func() { _ = os.Remove(settingsPath()) }) + + cfg, c, port := runArgs(t, newFake) + ctx, cancel := context.WithCancel(context.Background()) + + done := make(chan error, 1) + go func() { done <- run(ctx, logger.Test(t), "cron", cfg, c) }() + + require.Eventually(t, func() bool { return serving(int(port)) }, 30*time.Second, 5*time.Millisecond, + "the run should be serving while it is up") + + res, err := http.Post(fmt.Sprintf("http://localhost:%d%s", port, reloadPath()), "", nil) + require.NoError(t, err) + require.NoError(t, res.Body.Close()) + assert.Equal(t, http.StatusOK, res.StatusCode) + + cancel() + require.NoError(t, <-done) +} + +// TestUnderPluginHost covers the cookie go-plugin's Serve insists on, which is what decides whether +// a run hands the process to a host or waits for a signal. +// +// The check is necessary rather than defensive: Serve exits the process when the cookie is absent, +// so a standalone binary that called it anyway would die on startup. The other branch is not tested +// here - plugin.Serve takes over the process's stdio and handshake, so a test calling it would be +// testing go-plugin rather than this. +func TestUnderPluginHost(t *testing.T) { + h := loop.EmptyHandshakeConfig() + + t.Run("absent", func(t *testing.T) { + t.Setenv(h.MagicCookieKey, "") + assert.False(t, underPluginHost()) + }) + + t.Run("wrong value", func(t *testing.T) { + t.Setenv(h.MagicCookieKey, "not-the-cookie") + assert.False(t, underPluginHost()) + }) + + t.Run("present", func(t *testing.T) { + t.Setenv(h.MagicCookieKey, h.MagicCookieValue) + assert.True(t, underPluginHost()) + }) +} + +// assertPortFree reports whether nothing is listening on port, which is how a stopped web server is +// told from a running one. +func assertPortFree(t *testing.T, port uint16) { + t.Helper() + + l, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + if assert.NoError(t, err, "port %d should be free", port) { + require.NoError(t, l.Close()) + } +} diff --git a/libs/capability/server.go b/libs/capability/server.go new file mode 100644 index 000000000..96581d1a1 --- /dev/null +++ b/libs/capability/server.go @@ -0,0 +1,142 @@ +package capability + +import ( + "context" + "errors" + "fmt" + "net" + "strconv" + "sync" + "sync/atomic" + + "google.golang.org/grpc" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" +) + +// defaultHost is where a server binds and is advertised unless told otherwise. +const defaultHost = "localhost" + +// grpcConfig is where the gRPC servers this process opens bind and are advertised. +type grpcConfig struct { + AdvertiseHost string `usage:"host the gRPC servers this process opens bind to and are advertised at; empty binds every interface"` + + // StartPort is where the factory's ports begin. Zero, the default, means every server asks + // the OS for a free port instead. + StartPort uint16 `usage:"first port the gRPC servers this process opens bind to, incrementing per server; 0 asks the OS for a free port for each of them"` +} + +type server struct { + services.Service + eng *services.Engine + + grpc *grpc.Server + listener net.Listener + started atomic.Bool +} + +// newServer binds address and returns a server for it. address is host:port; port 0 asks the OS +// for a free one, which is logged once bound. +func newServer(ctx context.Context, lggr logger.Logger, address string) (*server, error) { + var lc net.ListenConfig + listener, err := lc.Listen(ctx, "tcp", address) + if err != nil { + return nil, fmt.Errorf("failed to listen on %s: %w", address, err) + } + + s := &server{grpc: grpc.NewServer(), listener: listener} + s.Service, s.eng = services.Config{ + Name: "GRPCServer", + Start: s.start, + // No Close hook: stopping the server is what releases the Serve goroutine, and the engine + // waits for its goroutines before it would run the hook - so the stop lives in Close + // itself, ahead of the handoff. + }.NewServiceEngine(lggr) + + s.eng.Infow(fmt.Sprintf("Bound gRPC to port %s", s.port()), "address", s.address()) + return s, nil +} + +func (s *server) grpcServer() *grpc.Server { return s.grpc } + +// address is the grpc.NewClient target for this server, which is what a caller announcing itself +// hands out. It is the address as bound, so a server on port 0 reports the port it actually got. +func (s *server) address() string { return s.listener.Addr().String() } + +// port is the port this server bound, as a string. +func (s *server) port() string { + if _, port, err := net.SplitHostPort(s.address()); err == nil { + return port + } + if tcp, ok := s.listener.Addr().(*net.TCPAddr); ok { + return strconv.Itoa(tcp.Port) + } + return "unknown" +} + +func (s *server) start(context.Context) error { + s.started.Store(true) + s.eng.Go(func(context.Context) { + if err := s.grpc.Serve(s.listener); err != nil && !errors.Is(err, grpc.ErrServerStopped) { + s.eng.Errorw("gRPC server stopped", "err", err) + } + }) + return nil +} + +// Close stops serving and releases the port, whether or not the server ever started. +// +// The stop comes before the handoff to the engine, not in its Close hook, because the engine +// closes in the other order: it waits for its goroutines before running the hook, and the Serve +// goroutine returns only once the server is stopped. Stopping from the hook would deadlock the +// two against each other. +// +// The two cases differ because the listener is opened by the constructor: a server that was +// started has the engine to unwind once serving has stopped; one that was not has no engine state +// at all, and would otherwise hold the port until the process exits - which is exactly the case a +// caller hits when it binds a server and then fails before starting it. +func (s *server) Close() error { + if s.started.Load() { + s.grpc.GracefulStop() + return s.Service.Close() + } + s.grpc.Stop() + return s.listener.Close() +} + +// serverFactory makes gRPC servers, one per thing that has to be told apart by address - see the +// file comment. Each call binds a port immediately, so the caller can announce the address before +// anything is serving on it. +// +// One factory hands out one run of ports: the ports are the process's, so a second counter would +// have two servers try to bind the same one. +type serverFactory struct { + host string + startPort uint16 + + mu sync.Mutex + opened uint16 // servers made so far, which is the offset from startPort +} + +// new returns a server bound to the next port on the configured host. lggr names it, and the +// caller names it after whatever it serves, so a process with several says which is which. +func (f *serverFactory) new(ctx context.Context, lggr logger.Logger) (*server, error) { + return newServer(ctx, lggr, net.JoinHostPort(f.host, strconv.Itoa(int(f.nextPort())))) +} + +// nextPort is the port the next server binds. +// +// Zero is not a port but a request for any free one, so it is handed out as-is however many +// servers ask: incrementing it would turn "any port" into a deliberate 1, 2, 3, which are neither +// free nor wanted. +func (f *serverFactory) nextPort() uint16 { + if f.startPort == 0 { + return 0 + } + f.mu.Lock() + defer f.mu.Unlock() + port := f.startPort + f.opened + f.opened++ + return port +} diff --git a/libs/capability/settings.go b/libs/capability/settings.go new file mode 100644 index 000000000..750d88b79 --- /dev/null +++ b/libs/capability/settings.go @@ -0,0 +1,117 @@ +package capability + +// CRE settings, and the limits resolved out of them. +// +// A capability binary runs under the empty LOOP: the node supervises its liveness over go-plugin but +// exposes no RPCs to it, so settings reach it through the filesystem instead. The node dumps each +// update to a conventional path and then hits this process's reload endpoint; both share a +// container, so os.TempDir() resolves to the same place on either side. +// +// The constants below are that convention, and are a copy of the ones in standalone/capability +// rather than a reference to them - importing that package would be a cycle, since it reaches back +// into the bootstrapper that imports this one. They have to keep the same values: the node writes +// the file, this reads it. + +import ( + "errors" + "fmt" + "io/fs" + "net/http" + "os" + "path/filepath" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/loop" + "github.com/smartcontractkit/chainlink-common/pkg/settings/cresettings" + "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + "github.com/smartcontractkit/chainlink-common/pkg/types/core" +) + +const ( + // settingsDirName names the directory, under os.TempDir(), that CRE settings are dumped to. + settingsDirName = "cre_limits" + + // settingsFileName is the name of the file CRE settings are dumped to, and the path suffix of + // the reload endpoint: /reload/. + // + // A limit's effective value is resolved out of this same payload, so there is no separate limits + // file: reloading this one reloads limits too. + settingsFileName = "settings.txt" + + // reloadPathPrefix is the route prefix reload requests are served on. + reloadPathPrefix = "/reload/" +) + +// settingsPath is the file CRE settings are dumped to. Both sides resolve it the same way, which is +// the whole contract: the node writes it, this process reads it. +func settingsPath() string { return filepath.Join(os.TempDir(), settingsDirName, settingsFileName) } + +// reloadPath is the route the node hits after dumping new settings: /reload/settings.txt. +func reloadPath() string { return reloadPathPrefix + settingsFileName } + +// reloadHandler re-reads the settings file and swaps it in. 200 means every limit in this process +// now resolves against the new settings; 500 means none of them do and the previous settings are +// still in force. +func reloadHandler(lggr logger.Logger, settings *loop.AtomicSettings, path string) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + if err := loadSettings(settings, path); err != nil { + lggr.Errorw("Failed to reload settings", "err", err, "path", path) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + lggr.Infow("Reloaded settings", "path", path) + fmt.Fprintln(w, "ok") + } +} + +// newSettings builds this process's settings, seeded from the dumped file if the node has already +// written one. +func newSettings(lggr logger.Logger) (*loop.AtomicSettings, error) { + s := &loop.AtomicSettings{Lggr: lggr} + s.SetGetter(cresettings.DefaultGetter) + + if err := loadSettings(s, settingsPath()); err != nil { + return nil, err + } + return s, nil +} + +// loadSettings reads the dumped settings file into s. +// +// A missing file is not an error: nothing has been dumped yet, so s keeps the getter it was built +// with and every limit resolves to its compiled-in default. That is the same state a LOOP starts in +// before its first update arrives. +func loadSettings(s *loop.AtomicSettings, path string) error { + b, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("failed to read settings file %s: %w", path, err) + } + + // Hash is left empty: the node owns the hash of an update, and what is on disk is only ever a + // copy of one. Nothing downstream compares it. + if err := s.Store(core.SettingsUpdate{Settings: string(b)}); err != nil { + return fmt.Errorf("failed to apply settings from %s: %w", path, err) + } + return nil +} + +// newLimitsFactory builds the limits factory over settings. +// +// AtomicSettings is a settings.Getter rather than a settings.Registry, so the factory polls it +// rather than subscribing - which is why swapping the getter is enough to make every limit in the +// process follow a reload. +// +// The meter is read here rather than captured earlier, so it has to be built after telemetry has +// installed itself: an instrument resolves its meter once, and one made against the noop client +// records nothing for the life of the process. +func newLimitsFactory(lggr logger.Logger, settings *loop.AtomicSettings) limits.Factory { + return limits.Factory{ + Settings: settings, + Meter: beholder.GetMeter(), + Logger: logger.Named(lggr, "Limits"), + } +} diff --git a/libs/capability/settings_test.go b/libs/capability/settings_test.go new file mode 100644 index 000000000..99292664e --- /dev/null +++ b/libs/capability/settings_test.go @@ -0,0 +1,102 @@ +package capability + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/settings/cresettings" +) + +// TestLoadSettingsWithoutAFile covers the state a binary starts in before the node has dumped +// anything: every limit resolves to its compiled-in default, which is the same state a LOOP is in +// before its first update arrives. +func TestLoadSettingsWithoutAFile(t *testing.T) { + s, err := newSettings(logger.Test(t)) + require.NoError(t, err) + require.NotNil(t, s) + + require.NoError(t, loadSettings(s, filepath.Join(t.TempDir(), "nothing-here.txt")), + "a missing file is not an error") +} + +// TestLoadSettingsReportsAnUnreadableFile covers the other side: a file that is there and cannot be +// used is a failure to start, not something to carry on past with defaults. +func TestLoadSettingsReportsAnUnreadableFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "settings.txt") + require.NoError(t, os.WriteFile(path, []byte("not a settings payload"), 0o600)) + + s, err := newSettings(logger.Test(t)) + require.NoError(t, err) + + require.ErrorContains(t, loadSettings(s, path), path) +} + +// TestSettingsPathIsTheSharedConvention pins the contract with the node: it writes this file and +// this process reads it, so the two have to resolve the same path. +// +// The constants are a copy of standalone/capability's - importing that package would be a cycle - +// so nothing but a test stops them drifting apart. +func TestSettingsPathIsTheSharedConvention(t *testing.T) { + assert.Equal(t, filepath.Join(os.TempDir(), "cre_limits", "settings.txt"), settingsPath()) + assert.Equal(t, "/reload/", reloadPathPrefix) +} + +// TestLimitsFactoryIsBuiltOverTheSettings covers the wiring the whole thing exists for: a limit made +// by this factory reads the settings this process holds, so a reload reaches it. +func TestLimitsFactoryIsBuiltOverTheSettings(t *testing.T) { + s, err := newSettings(logger.Test(t)) + require.NoError(t, err) + + f := newLimitsFactory(logger.Test(t), s) + + assert.Same(t, s, f.Settings, "the factory should poll the settings this process holds") + assert.NotNil(t, f.Meter) + assert.NotNil(t, f.Logger) +} + +// TestReloadHandlerSwapsTheSettings covers the endpoint the node hits after dumping new settings: a +// limit made from this process's factory resolves against the new payload on its next use, without +// anything having to be told. +func TestReloadHandlerSwapsTheSettings(t *testing.T) { + s, err := newSettings(logger.Test(t)) + require.NoError(t, err) + + // A global-scope setting, so no tenant is needed to resolve it. The compiled-in default is 0s. + f := newLimitsFactory(logger.Test(t), s) + limit, err := f.MakeTimeLimiter(cresettings.Default.TriggerRegistrationStatusUpdateTimeout) + require.NoError(t, err) + + path := filepath.Join(t.TempDir(), "settings.txt") + require.NoError(t, os.WriteFile(path, + []byte("[global]\nTriggerRegistrationStatusUpdateTimeout = \"15s\"\n"), 0o600)) + + w := httptest.NewRecorder() + reloadHandler(logger.Test(t), s, path)(w, httptest.NewRequest(http.MethodPost, reloadPath(), nil)) + require.Equal(t, http.StatusOK, w.Code) + + got, err := limit.Limit(t.Context()) + require.NoError(t, err) + assert.Equal(t, 15*time.Second, got, "the limit should resolve against the reloaded settings") +} + +// TestReloadHandlerKeepsThePreviousSettingsOnFailure covers the node's signal to retry later: a +// payload that cannot be applied is a 500, and what was in force stays in force. +func TestReloadHandlerKeepsThePreviousSettingsOnFailure(t *testing.T) { + s, err := newSettings(logger.Test(t)) + require.NoError(t, err) + + path := filepath.Join(t.TempDir(), "settings.txt") + require.NoError(t, os.WriteFile(path, []byte("not a settings payload"), 0o600)) + + w := httptest.NewRecorder() + reloadHandler(logger.Test(t), s, path)(w, httptest.NewRequest(http.MethodPost, reloadPath(), nil)) + assert.Equal(t, http.StatusInternalServerError, w.Code) +} diff --git a/libs/capability/telemetry.go b/libs/capability/telemetry.go new file mode 100644 index 000000000..f78d70548 --- /dev/null +++ b/libs/capability/telemetry.go @@ -0,0 +1,271 @@ +package capability + +import ( + "context" + "fmt" + "os" + "strings" + + prombridge "go.opentelemetry.io/contrib/bridges/prometheus" + "go.opentelemetry.io/otel/attribute" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/loop" + "github.com/smartcontractkit/chainlink-common/pkg/services" +) + +const ( + // Legacy env prefixes for the two map-valued telemetry settings. pkg/config/flags has no map + // support, so those are []string of key=value pairs here; a host sets one env var per entry + // instead, and envPairs still picks those up. See TelemetryConfig.Attributes. + envTelemetryAttributePrefix = "CL_TELEMETRY_ATTRIBUTE_" + envTelemetryAuthHeaderPrefix = "CL_TELEMETRY_AUTH_HEADER_" +) + +// TelemetryConfig is the beholder client's configuration: an empty Endpoint leaves telemetry off, and the global noop client in place, so +// instruments created by services record nothing. +type TelemetryConfig struct { + Endpoint string `usage:"OTLP gRPC endpoint telemetry is exported to; telemetry is disabled when unset"` + InsecureConnection bool `usage:"export telemetry over an insecure connection"` + CACertFile string `usage:"CA certificate file used to verify the telemetry endpoint"` + + // Attributes and AuthHeaders are key=value pairs ("env=staging") rather than maps, since + // flags cannot bind a map field. Entries from the legacy CL_TELEMETRY_ATTRIBUTE_ and + // CL_TELEMETRY_AUTH_HEADER_ env vars are merged in on top of these, so a plugin host + // setting one env var per entry keeps working. + Attributes []string `usage:"extra telemetry resource attributes, as key=value pairs" example:"['env=staging']"` + AuthHeaders []string `usage:"telemetry auth headers, as key=value pairs" flagdocs:"noexample"` + + AuthPubKeyHex string `usage:"public key the telemetry auth headers are derived from"` + AuthHeadersTTL commonconfig.Duration `usage:"how long generated telemetry auth headers are valid for"` + PrometheusBridgeEnabled bool `usage:"feed metrics registered on the prometheus registry into the telemetry pipeline"` +} + +// TracingConfig is the OTLP tracing configuration. Traces go to the telemetry endpoint, so Enabled +// does nothing unless TelemetryConfig.Endpoint is set too. +type TracingConfig struct { + Enabled bool `usage:"export traces to the telemetry endpoint"` + SamplingRatio float64 `usage:"fraction of traces sampled, from 0 to 1"` + TLSCertFile string `usage:"TLS certificate file used by the trace exporter"` +} + +// ChipIngressConfig points the beholder client's chip ingress emitter at an endpoint. Emitting is +// enabled by setting one. +type ChipIngressConfig struct { + Endpoint string `usage:"chip ingress gRPC endpoint; the emitter is disabled when unset"` + InsecureConnection bool `usage:"connect to chip ingress over an insecure connection"` +} + +// newTelemetry builds the beholder client and the service that owns it. +func newTelemetry(lggr logger.Logger, obs *observability) (*telemetryService, error) { + if obs.telemetry.Endpoint == "" { + return noopTelemetry(lggr), nil + } + + cfg, err := beholderConfig(lggr, obs) + if err != nil { + return nil, err + } + + client, err := beholder.NewClient(cfg) + if err != nil { + return nil, fmt.Errorf("failed to create beholder client: %w", err) + } + + return newTelemetryService(lggr, client, installGlobally(client)), nil +} + +func startTelemetry(ctx context.Context, lggr logger.Logger, obs *observability) (*telemetryService, error) { + telemetry, err := newTelemetry(lggr, obs) + if err != nil { + return nil, err + } + + return telemetry, telemetry.Start(ctx) +} + +// telemetryService is the beholder client and the process-global it is installed as, as one service +// - so that everything starting telemetry changes is undone by closing it. +// +// Installing happens when this is built rather than when it starts - see newTelemetry - so what +// closing undoes is something that was already done. The two are not symmetric, and cannot be: a +// capability binds its instruments while it is being constructed, which is before the root that +// starts this exists. +// +// The client is a sub-service rather than something this closes itself, which is what gets the +// order right. Sub-services start before the Start hook and close after the Close hook, so the +// client is running by the time it becomes the process's, and is still open when the globals are +// pointed back at the previous one - a late measurement reaches a live pipeline rather than a +// closed one. +type telemetryService struct { + services.Service + + // client is the beholder client this owns. The health checker mirrors itself through the same + // meter, so it has to be reachable from outside - which is the only reason it is a field. + client *beholder.Client +} + +// noopTelemetry is telemetry nobody configured: nothing to start, nothing to close, and no client +// behind it. The process keeps the noop beholder client it already had. +func noopTelemetry(lggr logger.Logger) *telemetryService { + t := &telemetryService{} + t.Service, _ = services.Config{Name: "Telemetry"}.NewServiceEngine(lggr) + return t +} + +func newTelemetryService(lggr logger.Logger, client *beholder.Client, restore func()) *telemetryService { + t := &telemetryService{client: client} + t.Service, _ = services.Config{ + Name: "Telemetry", + Close: func() error { restore(); return nil }, + NewSubServices: func(logger.Logger) []services.Service { + return []services.Service{client} + }, + }.NewServiceEngine(lggr) + return t +} + +// installGlobally makes client the process's beholder client, and returns what puts back the one +// that was there before. +// +// The global is the only lasting change starting telemetry makes - everything else it touches is +// owned by the client itself - so this is where reversing it has to be expressed. The otel +// providers are re-pointed on the way back as well as on the way in, since they are derived from +// whichever client is global and would otherwise keep pointing at the one being taken away. +func installGlobally(client *beholder.Client) func() { + previous := beholder.GetClient() + + beholder.SetClient(client) + beholder.SetGlobalOtelProviders() + + return func() { + beholder.SetClient(previous) + beholder.SetGlobalOtelProviders() + } +} + +// beholderConfig is the telemetry, tracing and chip ingress settings as the one client takes them. +func beholderConfig(lggr logger.Logger, obs *observability) (beholder.Config, error) { + cfg := beholder.DefaultConfig() + cfg.OtelExporterGRPCEndpoint = obs.telemetry.Endpoint + cfg.InsecureConnection = obs.telemetry.InsecureConnection + cfg.CACertFile = obs.telemetry.CACertFile + + attributes, err := envPairs(envTelemetryAttributePrefix, "telemetry.attributes", obs.telemetry.Attributes) + if err != nil { + return beholder.Config{}, err + } + for k, v := range attributes { + cfg.ResourceAttributes = append(cfg.ResourceAttributes, attribute.String(k, v)) + } + + cfg.AuthHeaders, err = envPairs(envTelemetryAuthHeaderPrefix, "telemetry.auth-headers", obs.telemetry.AuthHeaders) + if err != nil { + return beholder.Config{}, err + } + cfg.AuthPublicKeyHex = obs.telemetry.AuthPubKeyHex + cfg.AuthHeadersTTL = obs.telemetry.AuthHeadersTTL.Duration() + + // Logs already reach their destination via stderr (parsed by the plugin host when under one); + // don't stream them a second time. + cfg.LogStreamingEnabled = false + + if obs.telemetry.PrometheusBridgeEnabled { + // Feeds metrics already registered on the global prometheus registry (e.g. via promauto, + // like the health checker's) into the same OTLP pipeline, so they don't need a separate + // scrape target. + cfg.MetricProducers = append(cfg.MetricProducers, prombridge.NewMetricProducer()) + } + + cfg.ChipIngressEmitterGRPCEndpoint = obs.chipIngress.Endpoint + cfg.ChipIngressEmitterEnabled = obs.chipIngress.Endpoint != "" + cfg.ChipIngressInsecureConnection = obs.chipIngress.InsecureConnection + + if obs.tracing.Enabled { + tracing := loop.TracingConfig{ + Enabled: true, + CollectorTarget: obs.telemetry.Endpoint, + SamplingRatio: obs.tracing.SamplingRatio, + TLSCertPath: obs.tracing.TLSCertFile, + // Not optional, despite reading like it. The exporter's dialer calls this on every + // failed dial without checking it is set, so leaving it nil turns an unreachable + // collector - a network blip, a collector restarting - into a nil dereference on a + // background goroutine, which takes the process with it. + OnDialError: func(err error) { + logger.Sugared(lggr).Errorw("Failed to dial the tracing collector", + "err", err, "target", obs.telemetry.Endpoint) + }, + } + if cfg.AuthHeaders != nil { + tracing.AuthHeaders = cfg.AuthHeaders + } + + exporter, err := tracing.NewSpanExporter() + if err != nil { + return beholder.Config{}, fmt.Errorf("failed to setup tracing exporter: %w", err) + } + cfg.TraceSpanExporter = exporter + cfg.TraceSampleRatio = tracing.SamplingRatio + } + + // Per the OTEL specification, histogram buckets must be defined when the client is created, so + // the views cannot be applied any later than this - which is why they are handed to the binary + // rather than asked of the capability, whose constructor has not run yet. See WithOtelViews. + cfg.MetricViews = obs.otelViews + + return cfg, nil +} + +// envPairs merges the key=value pairs of a []string setting with the legacy one-env-var-per-entry +// form a plugin host encodes maps in (loop.EnvConfig.AsCmdEnv): PREFIX_SOME_KEY=value becomes +// SOME_KEY=value. The setting wins on conflict, being the more specific source. Returns nil when +// neither supplies anything. +func envPairs(envPrefix, setting string, pairs []string) (map[string]string, error) { + fromSetting, err := parsePairs(setting, pairs) + if err != nil { + return nil, err + } + + merged := envMap(envPrefix) + if merged == nil { + return fromSetting, nil + } + for k, v := range fromSetting { + merged[k] = v + } + return merged, nil +} + +// envMap collects env vars starting with prefix into a map, with the prefix stripped from the keys. +// Returns nil when none are set. +func envMap(prefix string) map[string]string { + var m map[string]string + for _, env := range os.Environ() { + if key, value, found := strings.Cut(env, "="); found && strings.HasPrefix(key, prefix) { + if m == nil { + m = make(map[string]string) + } + m[strings.TrimPrefix(key, prefix)] = value + } + } + return m +} + +// parsePairs turns key=value strings into a map, erroring on an entry without a "=". Values may +// themselves contain "=", so only the first one separates. +func parsePairs(setting string, pairs []string) (map[string]string, error) { + if len(pairs) == 0 { + return nil, nil + } + m := make(map[string]string, len(pairs)) + for _, pair := range pairs { + key, value, found := strings.Cut(pair, "=") + if !found || key == "" { + return nil, fmt.Errorf("invalid %s entry %q: expected key=value", setting, pair) + } + m[key] = value + } + return m, nil +} diff --git a/libs/capability/telemetry_test.go b/libs/capability/telemetry_test.go new file mode 100644 index 000000000..164cab1b1 --- /dev/null +++ b/libs/capability/telemetry_test.go @@ -0,0 +1,401 @@ +package capability + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink-common/pkg/logger" +) + +// settingsRoot builds the command tree the way RunErr does, through the same registration, and +// returns what the flags are bound to. +// +// The registration is the production one, so what these tests read is what a binary gets. +func settingsRoot(t *testing.T) (*cobra.Command, *config) { + t.Helper() + + root := &cobra.Command{Use: "cron"} + root.PersistentFlags().String("config", "", "Path to config file") + + // The production defaults, so what these read is what a binary gets rather than a zero value. + cfg := defaultConfig() + require.NoError(t, cfg.bind(root)) + return root, cfg +} + +// decoded runs args through the command tree and returns the settings as they were decoded. +func decoded(t *testing.T, args ...string) config { + t.Helper() + + root, cfg := settingsRoot(t) + root.AddCommand(&cobra.Command{Use: "run", RunE: func(*cobra.Command, []string) error { return nil }}) + + args = append([]string{"run"}, args...) + root.SetArgs(args) + require.NoError(t, root.Execute()) + return *cfg +} + +// TestTelemetryFlags pins the surface the beholder client's settings have: a flag per field, named +// after the service that owns it. A binary gets these without asking, because every binary has them. +func TestObservabilityFlags(t *testing.T) { + root, _ := settingsRoot(t) + + for _, name := range []string{ + "telemetry.endpoint", + "telemetry.insecure-connection", + "telemetry.ca-cert-file", + "telemetry.attributes", + "telemetry.auth-headers", + "telemetry.auth-pub-key-hex", + "telemetry.auth-headers-ttl", + "telemetry.prometheus-bridge-enabled", + + "tracing.enabled", + "tracing.sampling-ratio", + "tracing.tls-cert-file", + + "chip-ingress.endpoint", + "chip-ingress.insecure-connection", + + "pyroscope.server-address", + "pyroscope.auth-token", + "pyroscope.environment", + + "http.port", + + "capabilities.proxy-url", + "capabilities.capability-don-id", + } { + assert.NotNil(t, root.PersistentFlags().Lookup(name), "missing flag --%s", name) + } +} + +// TestCapabilitiesDecodeFromFlags covers the sibling section: what a capability binary needs from +// the node it runs beside, which is not observability and so is configured separately. +func TestCapabilitiesDecodeFromFlags(t *testing.T) { + cfg := decoded(t, "--capabilities.proxy-url", "dns:///registry.internal:9000", + "--capabilities.capability-don-id", "7") + + assert.Equal(t, "dns:///registry.internal:9000", cfg.capabilities.ProxyURL) + assert.Equal(t, uint32(7), cfg.capabilities.CapabilityDonID) +} + +// TestCapabilitiesAreOptional covers both settings being optional: a binary with no node behind it +// starts, and resolves only the capabilities it hosts. +// +// --http.port is the one thing a run still cannot do without, which is what the args below are. +func TestCapabilitiesAreOptional(t *testing.T) { + root, cfg := settingsRoot(t) + root.AddCommand(&cobra.Command{Use: "run", RunE: func(*cobra.Command, []string) error { return nil }}) + root.SetArgs([]string{"run", "--http.port", "1"}) + + require.NoError(t, root.Execute()) + assert.Empty(t, cfg.capabilities.ProxyURL) + assert.Zero(t, cfg.capabilities.CapabilityDonID) +} + +// TestObservabilityDefaults pins the settings that are not their zero value, since those are the +// ones an operator gets without asking. +func TestObservabilityDefaults(t *testing.T) { + root, _ := settingsRoot(t) + + sampling := root.PersistentFlags().Lookup("tracing.sampling-ratio") + require.NotNil(t, sampling) + assert.Equal(t, "1", sampling.DefValue, "sampling every trace is the default") + + port := root.PersistentFlags().Lookup("http.port") + require.NotNil(t, port) + assert.Equal(t, "8080", port.DefValue) +} + +// TestBinaryStartsWithNoSettingsAtAll is what defaulting the port bought: nothing has to be +// configured for a run to be a valid one. +// +// It is the whole surface in one assertion - every setting this binary has is now either defaulted +// or genuinely optional - so a flag gaining `validate:"required"` fails here. +func TestBinaryStartsWithNoSettingsAtAll(t *testing.T) { + root, cfg := settingsRoot(t) + root.AddCommand(&cobra.Command{Use: "run", RunE: func(*cobra.Command, []string) error { return nil }}) + root.SetArgs([]string{"run"}) + root.SilenceUsage, root.SilenceErrors = true, true + + require.NoError(t, root.Execute()) + assert.Equal(t, uint16(defaultHTTPPort), cfg.observability.http.Port) +} + +// TestTelemetryDecodesFromFlags is the point of registering them: what an operator types reaches the +// struct the beholder client is built from. +func TestObservabilityDecodesFromFlags(t *testing.T) { + cfg := decoded(t, + "--tracing.enabled", "true", + "--tracing.sampling-ratio", "0.25", + "--tracing.tls-cert-file", "/certs/trace.pem", + "--chip-ingress.endpoint", "chip:9000", + "--chip-ingress.insecure-connection", "true", + "--pyroscope.server-address", "pyro:4040", + "--pyroscope.environment", "staging", + "--telemetry.endpoint", "otel:4317", + "--telemetry.insecure-connection", "true", + "--telemetry.attributes", "env=staging", + "--telemetry.auth-headers-ttl", "5m", + "--telemetry.prometheus-bridge-enabled", "true", + ) + + assert.Equal(t, "otel:4317", cfg.observability.telemetry.Endpoint) + assert.True(t, cfg.observability.telemetry.InsecureConnection) + assert.Equal(t, []string{"env=staging"}, cfg.observability.telemetry.Attributes) + assert.Equal(t, 5*time.Minute, cfg.observability.telemetry.AuthHeadersTTL.Duration()) + assert.True(t, cfg.observability.telemetry.PrometheusBridgeEnabled) + + assert.True(t, cfg.observability.tracing.Enabled) + assert.InDelta(t, 0.25, cfg.observability.tracing.SamplingRatio, 0) + assert.Equal(t, "/certs/trace.pem", cfg.observability.tracing.TLSCertFile) + + assert.Equal(t, "chip:9000", cfg.observability.chipIngress.Endpoint) + assert.True(t, cfg.observability.chipIngress.InsecureConnection) + + assert.Equal(t, "pyro:4040", cfg.observability.pyroscope.ServerAddress) + assert.Equal(t, "staging", cfg.observability.pyroscope.Environment) +} + +// TestChipIngressFoldsIntoTheBeholderClient covers the shape of these three: chip ingress and +// tracing are configured separately but exported through the one client, so what turns them on is +// a field on its config rather than a service of their own. +func TestChipIngressFoldsIntoTheBeholderClient(t *testing.T) { + off, err := beholderConfig(logger.Test(t), &observability{telemetry: TelemetryConfig{Endpoint: "otel:4317"}}) + require.NoError(t, err) + assert.False(t, off.ChipIngressEmitterEnabled, "an unset endpoint leaves the emitter off") + + on, err := beholderConfig(logger.Test(t), &observability{ + telemetry: TelemetryConfig{Endpoint: "otel:4317"}, + chipIngress: ChipIngressConfig{Endpoint: "chip:9000", InsecureConnection: true}, + }) + require.NoError(t, err) + assert.True(t, on.ChipIngressEmitterEnabled, "setting an endpoint is what enables it") + assert.Equal(t, "chip:9000", on.ChipIngressEmitterGRPCEndpoint) + assert.True(t, on.ChipIngressInsecureConnection) +} + +// TestTracingFoldsIntoTheBeholderClient is the same for traces: they go to the telemetry endpoint, +// so enabling them adds an exporter to the client rather than standing anything else up. +func TestTracingFoldsIntoTheBeholderClient(t *testing.T) { + lggr, _ := detachedLogger() + + off, err := beholderConfig(lggr, &observability{telemetry: TelemetryConfig{Endpoint: "otel:4317"}}) + require.NoError(t, err) + assert.Nil(t, off.TraceSpanExporter) + + on, err := beholderConfig(lggr, &observability{ + telemetry: TelemetryConfig{Endpoint: "otel:4317"}, + tracing: TracingConfig{Enabled: true, SamplingRatio: 0.25}, + }) + require.NoError(t, err) + require.NotNil(t, on.TraceSpanExporter, "enabling tracing should add an exporter") + assert.InDelta(t, 0.25, on.TraceSampleRatio, 0) + + // The exporter dials its collector on a goroutine of its own, so it has to be shut down or it + // outlives the test that made it. + require.NoError(t, on.TraceSpanExporter.Shutdown(context.Background())) +} + +// TestTracingSurvivesAnUnreachableCollector covers the dialer's error path, which is not optional: +// loop.TracingConfig calls OnDialError without checking it is set, so an exporter built without one +// turns an unreachable collector into a nil dereference that takes the process with it. +func TestTracingSurvivesAnUnreachableCollector(t *testing.T) { + lggr, logs := detachedLogger() + + // A port nothing is listening on, so the dial is guaranteed to fail. + cfg, err := beholderConfig(lggr, &observability{ + telemetry: TelemetryConfig{Endpoint: fmt.Sprintf("localhost:%d", freePort(t))}, + tracing: TracingConfig{Enabled: true, SamplingRatio: 1}, + }) + require.NoError(t, err) + require.NotNil(t, cfg.TraceSpanExporter) + + // Exporting is what makes it dial, and the dial is asynchronous - so the failure is waited for + // rather than asserted straight away. Reaching this log at all is the point: without + // OnDialError the same path is a nil dereference, which would take the test binary with it + // rather than failing this assertion. + _ = cfg.TraceSpanExporter.ExportSpans(context.Background(), nil) + + require.Eventually(t, func() bool { + return len(logs.FilterMessage("Failed to dial the tracing collector").All()) > 0 + }, 10*time.Second, 10*time.Millisecond, "the dial failure should be reported rather than swallowed") + + require.NoError(t, cfg.TraceSpanExporter.Shutdown(context.Background())) +} + +// TestTelemetryDecodesFromEnv covers why the namespace and field names are what they are: a +// go-plugin host sets CL_TELEMETRY_ENDPOINT, and a binary it starts has to pick that up without +// being passed a flag. +func TestObservabilityDecodesFromEnv(t *testing.T) { + t.Setenv("CL_TELEMETRY_ENDPOINT", "from-env:4317") + + assert.Equal(t, "from-env:4317", decoded(t).observability.telemetry.Endpoint) +} + +// TestBeholderConfigMergesLegacyEnvPairs covers the other half of that contract: a host encodes a +// map as one env var per entry, and the setting wins where both name the same key. +func TestBeholderConfigMergesLegacyEnvPairs(t *testing.T) { + t.Setenv(envTelemetryAttributePrefix+"region", "us-east") + t.Setenv(envTelemetryAttributePrefix+"env", "from-env") + + cfg, err := beholderConfig(logger.Test(t), &observability{ + telemetry: TelemetryConfig{Endpoint: "otel:4317", Attributes: []string{"env=from-setting"}}, + }) + require.NoError(t, err) + + attributes := map[string]string{} + for _, a := range cfg.ResourceAttributes { + attributes[string(a.Key)] = a.Value.AsString() + } + assert.Equal(t, "us-east", attributes["region"], "the env-only entry should survive") + assert.Equal(t, "from-setting", attributes["env"], "the setting should win over the env var") +} + +// TestWithOtelViewsReachesTheClient is the whole of what the option is for: the views a binary +// passes are on the config the beholder client is built from. +// +// They cannot arrive any later - the OTEL specification requires histogram buckets at client +// creation - which is why this is an option on Run rather than something the capability declares. +func TestWithOtelViewsReachesTheClient(t *testing.T) { + view := sdkmetric.NewView( + sdkmetric.Instrument{Name: "cron_capability_*"}, + sdkmetric.Stream{Aggregation: sdkmetric.AggregationExplicitBucketHistogram{Boundaries: []float64{1, 2, 3}}}, + ) + + cfg := defaultConfig() + cfg.observability.telemetry.Endpoint = "otel:4317" + WithOtelViews(view)(cfg) + + bcfg, err := beholderConfig(logger.Test(t), &cfg.observability) + require.NoError(t, err) + assert.Len(t, bcfg.MetricViews, 1, "the view should be on the config the client is built from") +} + +// TestWithOtelViewsDefaultsToNone covers a binary that passes none: the client keeps whatever +// beholder's own defaults are rather than being handed an empty list to mean something. +func TestWithOtelViewsDefaultsToNone(t *testing.T) { + obs := defaultObservability() + obs.telemetry.Endpoint = "otel:4317" + + cfg, err := beholderConfig(logger.Test(t), obs) + require.NoError(t, err) + assert.Empty(t, cfg.MetricViews) +} + +// TestRunUsesTheDecodedTelemetrySettings is the whole path in one: a flag on the command line +// reaches the config the beholder client is built from. +// +// A malformed attribute is what makes that visible without a reachable collector - building the +// config fails before a client is ever created - and an unregistered flag would have failed +// earlier, with cobra's "unknown flag" instead. +func TestRunUsesTheDecodedTelemetrySettings(t *testing.T) { + err := execute(t, logger.Test(t), newFake, "run", + "--telemetry.endpoint", "otel:4317", "--telemetry.attributes", "nope") + + require.ErrorContains(t, err, `invalid telemetry.attributes entry "nope": expected key=value`) +} + +// TestStartTelemetryWithoutAnEndpointChangesNothing is the off case: no endpoint means no client, +// so nothing is installed, nothing joins the root, and there is nothing to undo. +func TestStartTelemetryWithoutAnEndpointChangesNothing(t *testing.T) { + before := beholder.GetClient() + + telemetry, err := newTelemetry(logger.Test(t), &observability{}) + require.NoError(t, err) + + // A service that does nothing rather than a nil, so the caller has one thing to hand the root + // either way. No client behind it: the process keeps the noop beholder client it already had. + require.NotNil(t, telemetry) + assert.Nil(t, telemetry.client, "there is no client when telemetry is off") + assert.Same(t, before, beholder.GetClient(), "nothing should have been installed") + + require.NoError(t, telemetry.Start(t.Context())) + require.NoError(t, telemetry.Close()) + assert.Same(t, before, beholder.GetClient(), "and starting or closing it changes nothing") +} + +// TestTelemetryServiceRestoresTheGlobalWhenClosed is what making telemetry a service bought: what +// starting it changed about the process is undone by closing it, rather than by the run remembering +// to. +// TestTelemetryServiceRestoresTheGlobalWhenClosed is what making telemetry a service bought: the +// run hands it to the root and forgets about it, and closing it is what puts the process back. +func TestTelemetryServiceRestoresTheGlobalWhenClosed(t *testing.T) { + original := beholder.GetClient() + client := beholder.NewNoopClient() + require.NotSame(t, original, client) + + svc := newTelemetryService(logger.Test(t), client, installGlobally(client)) + require.Same(t, client, beholder.GetClient(), "installing happens when it is built, not when it starts") + + require.NoError(t, svc.Start(t.Context())) + require.NoError(t, svc.Close()) + assert.Same(t, original, beholder.GetClient(), "closing should put the previous client back") +} + +// TestInstallGloballyIsReversible is the reversibility itself: installing a client replaces the +// process's, and what comes back puts the previous one in its place. +// +// This is the only lasting change starting telemetry makes - everything else belongs to the client +// - so it is the thing worth pinning. It is tested here rather than through startBeholder because +// that would need a live OTLP endpoint to get as far as installing anything. +func TestInstallGloballyIsReversible(t *testing.T) { + original := beholder.GetClient() + client := beholder.NewNoopClient() + require.NotSame(t, original, client) + + restore := installGlobally(client) + assert.Same(t, client, beholder.GetClient(), "the new client should be the process's") + + restore() + assert.Same(t, original, beholder.GetClient(), "the previous client should be back") +} + +// TestInstallGloballyNests covers a second install on top of a first: each restore puts back what +// that install replaced, so unwinding in reverse arrives where it started. +func TestInstallGloballyNests(t *testing.T) { + original := beholder.GetClient() + first, second := beholder.NewNoopClient(), beholder.NewNoopClient() + + restoreFirst := installGlobally(first) + restoreSecond := installGlobally(second) + assert.Same(t, second, beholder.GetClient()) + + restoreSecond() + assert.Same(t, first, beholder.GetClient()) + + restoreFirst() + assert.Same(t, original, beholder.GetClient()) +} + +// TestRunLeavesTelemetryAsItFoundIt is the same guarantee seen from the binary: a run that started +// telemetry and finished has put the process back the way it was. +func TestRunLeavesTelemetryAsItFoundIt(t *testing.T) { + before := beholder.GetClient() + + require.NoError(t, execute(t, logger.Test(t), newFake, "run")) + + assert.Same(t, before, beholder.GetClient()) +} + +// detachedLogger is an observed logger that is not bound to t. +// +// A trace exporter dials its collector on a goroutine of its own and keeps retrying, so it outlives +// the test that made it - and logger.Test panics on anything logged after t has finished, which +// would make these tests fail for a reason that has nothing to do with what they check. +func detachedLogger() (logger.Logger, *observer.ObservedLogs) { + core, logs := observer.New(zapcore.ErrorLevel) + return logger.NewWithCores(core), logs +} diff --git a/libs/capability/webserver.go b/libs/capability/webserver.go new file mode 100644 index 000000000..32261e198 --- /dev/null +++ b/libs/capability/webserver.go @@ -0,0 +1,179 @@ +package capability + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "net/http/pprof" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + + "github.com/smartcontractkit/chainlink-common/pkg/beholder" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-common/pkg/services/otelhealth" + "github.com/smartcontractkit/chainlink-common/pkg/services/promhealth" +) + +// defaultHTTPPort is where the shared HTTP server listens unless it is told otherwise. +const defaultHTTPPort = 8080 + +// HTTPConfig is the shared HTTP server: /metrics, /debug/pprof, the health endpoints, and whatever +// routes a service registers on the mux while it is being built - it is not only prometheus's, so +// it is named for the transport it serves rather than for one of its handlers. +type HTTPConfig struct { + Port uint16 `usage:"port serving /metrics, /debug/pprof, /healthz, /readyz and any routes a service registers"` +} + +// newWebServer returns the shared HTTP server as a service. +func newWebServer(lggr logger.Logger, cfg HTTPConfig, mux *http.ServeMux, checker *services.HealthChecker) *webService { + mux.Handle("/metrics", promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{ + EnableOpenMetrics: true, + })) + + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + + mux.HandleFunc("/healthz", healthHandler(checker.IsHealthy)) + mux.HandleFunc("/readyz", healthHandler(checker.IsReady)) + + w := &webService{ + lggr: lggr, + port: cfg.Port, + server: &http.Server{ + Handler: mux, + // Reasonable default based on a typical prometheus poll interval of 15s. + ReadTimeout: 5 * time.Second, + }, + } + w.Service, _ = services.Config{ + Name: "WebServer", + Start: w.start, + Close: w.close, + }.NewServiceEngine(lggr) + return w +} + +// startWebServer builds the shared HTTP server and starts it - listening is what Start does. +func startWebServer(ctx context.Context, lggr logger.Logger, cfg HTTPConfig, mux *http.ServeMux, checker *services.HealthChecker) (*webService, error) { + w := newWebServer(lggr, cfg, mux, checker) + return w, w.Start(ctx) +} + +type webService struct { + services.Service + + lggr logger.Logger + port uint16 + server *http.Server + + listener net.Listener +} + +func (w *webService) start(ctx context.Context) error { + // An explicit listener resolves port 0 before Serve, so the chosen port can be logged. + var lc net.ListenConfig + listener, err := lc.Listen(ctx, "tcp", fmt.Sprintf(":%d", w.port)) + if err != nil { + return fmt.Errorf("failed to listen on port %d: %w", w.port, err) + } + w.listener = listener + + go func() { + if err := w.server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { + logger.Sugared(w.lggr).Errorw("Metrics and health server stopped", "err", err) + } + }() + + w.lggr.Infow("Serving metrics and health endpoints", "address", listener.Addr().String()) + return nil +} + +func (w *webService) close() error { + // closes the listener too + return w.server.Close() +} + +// newHealthChecker returns the health checker as a service. +// +// client is always a client, never nil: when telemetry is off it is the noop one that is global +// until something replaces it, so the otel hooks are configured either way and simply record +// nothing. +func newHealthChecker(lggr logger.Logger, client *beholder.Client, reporters []services.HealthReporter) (*healthService, error) { + cfg := promhealth.ConfigureHooks(services.HealthCheckerConfig{}) + newCfg, err := otelhealth.ConfigureHooks(cfg, client.Meter) + if err != nil { + return nil, fmt.Errorf("failed to configure health checker otel hooks: %w", err) + } + cfg = newCfg + + h := &healthService{checker: cfg.New(), reporters: reporters} + h.Service, _ = services.Config{ + Name: "HealthChecker", + Start: h.start, + Close: h.close, + }.NewServiceEngine(lggr) + return h, nil +} + +func startHealthChecker(ctx context.Context, lggr logger.Logger, client *beholder.Client, reporters []services.HealthReporter) (*healthService, error) { + h, err := newHealthChecker(lggr, client, reporters) + if err != nil { + return nil, err + } + return h, h.Start(ctx) +} + +type healthService struct { + services.Service + + // checker is made when this is built, so that whatever serves its view can be given it without + // waiting for this to start. + checker *services.HealthChecker + reporters []services.HealthReporter +} + +func (h *healthService) start(context.Context) error { + if err := h.checker.Start(); err != nil { + return fmt.Errorf("failed to start health checker: %w", err) + } + // Registering reads a reporter's health as it goes, which is why this is here rather than at + // construction: everything being reported on has to be running by now. + for _, r := range h.reporters { + if err := h.checker.Register(r); err != nil { + // Started but not recorded as started, so nothing else will stop it. + return errors.Join( + fmt.Errorf("failed to register %s with the health checker: %w", r.Name(), err), + h.checker.Close()) + } + } + return nil +} + +func (h *healthService) close() error { return h.checker.Close() } + +// healthHandler adapts a services.HealthChecker.IsHealthy/IsReady-shaped func into an HTTP handler: +// 200 with each check's status when ok, 503 and the failing checks' errors otherwise. +func healthHandler(check func() (bool, map[string]error)) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + ok, errs := check() + if !ok { + w.WriteHeader(http.StatusServiceUnavailable) + } + for name, err := range errs { + if err != nil { + fmt.Fprintf(w, "%s: %s\n", name, err) + } + } + if ok { + fmt.Fprintln(w, "ok") + } + } +} diff --git a/libs/go.mod b/libs/go.mod index d03260b46..da6f6d656 100644 --- a/libs/go.mod +++ b/libs/go.mod @@ -1,24 +1,30 @@ module github.com/smartcontractkit/capabilities/libs -go 1.26.4 +go 1.26.6 require ( github.com/cenkalti/backoff/v5 v5.0.3 + github.com/fullstorydev/grpcui v1.5.4 github.com/google/go-cmp v0.7.0 github.com/google/uuid v1.6.0 + github.com/grafana/pyroscope-go v1.2.8 github.com/hashicorp/go-plugin v1.8.0 + github.com/jhump/protoreflect v1.18.1 + github.com/prometheus/client_golang v1.23.2 github.com/shopspring/decimal v1.4.0 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260713185857-30ad2e76c0f4 - github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260707203317-661b54b51a33 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260827151019-9853beb3a544 + github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260804191526-b7a850ae7648 github.com/smartcontractkit/libocr v0.0.0-20260810200708-618b5bf7f342 - github.com/stretchr/testify v1.11.1 + github.com/spf13/cobra v1.8.1 + github.com/stretchr/testify v1.12.0 + go.opentelemetry.io/contrib/bridges/prometheus v0.68.0 go.opentelemetry.io/otel v1.44.0 go.opentelemetry.io/otel/metric v1.44.0 go.opentelemetry.io/otel/sdk/metric v1.44.0 go.uber.org/zap v1.27.1 - google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa - google.golang.org/grpc v1.83.1 - google.golang.org/protobuf v1.36.11 + google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5 + google.golang.org/grpc v1.83.2 + google.golang.org/protobuf v1.36.12 ) require ( @@ -30,10 +36,15 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.16.1 // indirect github.com/cloudevents/sdk-go/v2 v2.16.1 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.39.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/fatih/color v1.18.0 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fullstorydev/grpcurl v1.9.3 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.8 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-json-experiment/json v0.0.0-20250223041408-d3c622f1b874 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -43,28 +54,30 @@ require ( github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/grafana/otel-profiling-go v0.5.1 // indirect - github.com/grafana/pyroscope-go v1.2.8 // indirect github.com/grafana/pyroscope-go/godeltaprof v0.1.9 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/hashicorp/go-hclog v1.6.3 // indirect github.com/hashicorp/yamux v0.1.2 // indirect + github.com/iancoleman/strcase v0.3.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/invopop/jsonschema v0.13.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgx/v5 v5.9.2 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jhump/protoreflect/v2 v2.0.0-beta.2 // indirect github.com/jmoiron/sqlx v1.4.0 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.5 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lib/pq v1.10.9 // indirect github.com/mailru/easyjson v0.9.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-isatty v0.0.21 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mr-tron/base58 v1.2.0 // indirect @@ -72,25 +85,33 @@ require ( github.com/oklog/run v1.2.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/petermattis/goid v0.0.0-20260820044319-269ab09b5261 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.23.2 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.20.1 // indirect + github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/scylladb/go-reflectx v1.0.1 // indirect github.com/smartcontractkit/chain-selectors v1.0.104 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 // indirect github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b // indirect + github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe // indirect github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 // indirect github.com/smartcontractkit/freeport v0.1.3-0.20250716200817-cb5dfd0e369e // indirect github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 // indirect - github.com/stretchr/objx v0.5.2 // indirect + github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/spf13/viper v1.21.0 // indirect + github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect + github.com/stretchr/objx v0.5.3 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/bridges/prometheus v0.68.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.12.2 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.19.0 // indirect @@ -110,12 +131,13 @@ require ( go.uber.org/goleak v1.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect - golang.org/x/crypto v0.53.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.55.0 // indirect golang.org/x/exp v0.0.0-20260508232706-74f9aab9d74a // indirect - golang.org/x/net v0.55.0 // indirect - golang.org/x/sync v0.21.0 // indirect - golang.org/x/sys v0.46.0 // indirect - golang.org/x/text v0.38.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/libs/go.sum b/libs/go.sum index d15854361..b3f3edcd4 100644 --- a/libs/go.sum +++ b/libs/go.sum @@ -27,23 +27,38 @@ github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.16.1/go.mod h1:6Q+F2 github.com/cloudevents/sdk-go/v2 v2.16.1 h1:G91iUdqvl88BZ1GYYr9vScTj5zzXSyEuqbfE63gbu9Q= github.com/cloudevents/sdk-go/v2 v2.16.1/go.mod h1:v/kVOaWjNfbvc6tkhhlkhvLapj8Aa8kvXiH5GiOHCKI= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dominikbraun/graph v0.23.0 h1:TdZB4pPqCLFxYhdyMFb1TBdFxp8XLcJfTTBQucVPgCo= github.com/dominikbraun/graph v0.23.0/go.mod h1:yOjYyogZLY1LSG9E33JWZJiq5k83Qy2C6POAuiViluc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane/envoy v1.39.0 h1:1uwRDYPYG8BIBU9Mj1sUAebNmlM6beu/ZKKweSLDxk8= +github.com/envoyproxy/go-control-plane/envoy v1.39.0/go.mod h1:5e4ylfTZO723MEEFsCpSW4ZEBWR8mwkEyXfwJBTCZ9c= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fullstorydev/grpcui v1.5.4 h1:6B6bFF0E8FJkt6vz27PMVlzNlowMrIak1WVcQfecUgg= +github.com/fullstorydev/grpcui v1.5.4/go.mod h1:Gi4FWQpJ4gaxg6Fewwpc57N8GYPTCbXvPf6webX89pw= +github.com/fullstorydev/grpcurl v1.9.3 h1:PC1Xi3w+JAvEE2Tg2Gf2RfVgPbf9+tbuQr1ZkyVU3jk= +github.com/fullstorydev/grpcurl v1.9.3/go.mod h1:/b4Wxe8bG6ndAjlfSUjwseQReUDUvBJiFEB7UllOlUE= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-json-experiment/json v0.0.0-20250223041408-d3c622f1b874 h1:F8d1AJ6M9UQCavhwmO6ZsrYLfG8zVFWfEfMS2MXPkSY= github.com/go-json-experiment/json v0.0.0-20250223041408-d3c622f1b874/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -115,6 +130,10 @@ github.com/hashicorp/go-plugin v1.8.0 h1:ie8S6RRY8RvB2usYZv+AAZ/wBvx2AU5p5QeP5j/ github.com/hashicorp/go-plugin v1.8.0/go.mod h1:BExt6KEaIYx804z8k4gRzRLEvxKVb+kn0NMcihqOqb8= github.com/hashicorp/yamux v0.1.2 h1:XtB8kyFOyHXYVFnwT5C3+Bdo8gArse7j2AQ0DA0Uey8= github.com/hashicorp/yamux v0.1.2/go.mod h1:C+zze2n6e/7wshOZep2A70/aQU6QBRWJO/G6FT1wIns= +github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= +github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E= github.com/invopop/jsonschema v0.13.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -125,8 +144,10 @@ github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw= github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/jhump/protoreflect v1.17.0 h1:qOEr613fac2lOuTgWN4tPAtLL7fUSbuJL5X5XumQh94= -github.com/jhump/protoreflect v1.17.0/go.mod h1:h9+vUUL38jiBzck8ck+6G/aeMX8Z4QUY/NiJPwPNi+8= +github.com/jhump/protoreflect v1.18.1 h1:h4odAaLg9wyn7yHxMF7sSkJ7JfLwK1oy37/1Pi212GE= +github.com/jhump/protoreflect v1.18.1/go.mod h1:I2yar2oJEMf0k4EMryPzfV0tvGwN/SejJziYBOpETQo= +github.com/jhump/protoreflect/v2 v2.0.0-beta.2 h1:qZU+rEZUOYTz1Bnhi3xbwn+VxdXkLVeEpAeZzVXLY88= +github.com/jhump/protoreflect/v2 v2.0.0-beta.2/go.mod h1:4tnOYkB/mq7QTyS3YKtVtNrJv4Psqout8HA1U+hZtgM= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= @@ -136,8 +157,8 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -160,8 +181,8 @@ github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHP github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs= +github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= @@ -180,13 +201,15 @@ github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3v github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= -github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= -github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/petermattis/goid v0.0.0-20260820044319-269ab09b5261 h1:lcWAnrqr2nNfDiArwFNHCE4787Mw2tCdVSOXCru0/0E= +github.com/petermattis/goid v0.0.0-20260820044319-269ab09b5261/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= +github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= +github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -198,6 +221,9 @@ github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEy github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= +github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/scylladb/go-reflectx v1.0.1 h1:b917wZM7189pZdlND9PbIJ6NQxfDPfBvUaQ7cjj1iZQ= @@ -206,16 +232,18 @@ github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/smartcontractkit/chain-selectors v1.0.104 h1:/n9pPGM5W/+r1eHoWZv4VwX9LNS1af4+ICyhM8zKRNM= github.com/smartcontractkit/chain-selectors v1.0.104/go.mod h1:qy7whtgG5g+7z0jt0nRyii9bLND9m15NZTzuQPkMZ5w= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260713185857-30ad2e76c0f4 h1:KSg0EnUdefIGyR3Fa6/nXhWXaCUlMP5qFAsLHwzi6Fk= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260713185857-30ad2e76c0f4/go.mod h1:snfVBRRQTpC2x5O3bQHZe9SvJX5yv/SbG8oHkJTKLtE= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62 h1:o7vfwNQjQbMKQ9YsZFQOxvU7RMXD/wKnZsX5N9sDS3w= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260626151909-052e55e62e62/go.mod h1:HmUyH2oD9m+GRpKq7q3vuRnm1F2Uczf/Nd1v3ipMSK8= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260827151019-9853beb3a544 h1:u7Ma/0jEiMQR3viN7GHtL1aVRBk7AL6YgR/n0YvnCsc= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260827151019-9853beb3a544/go.mod h1:a1f3IeYqiHwmysfFDg83R2XS6mx+ObG8L5TBZf3cW6g= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 h1:uWEwl7i2ryuRVoV4DmIKm6mqYevf1lH/8cQYhw/JXko= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72/go.mod h1:UYcRMb4dZcoaIPgZJ3hckCySTqtJc9K4Q+tOKErwTq0= github.com/smartcontractkit/chainlink-protos/billing/go v0.0.0-20251024234028-0988426d98f4 h1:GCzrxDWn3b7jFfEA+WiYRi8CKoegsayiDoJBCjYkneE= github.com/smartcontractkit/chainlink-protos/billing/go v0.0.0-20251024234028-0988426d98f4/go.mod h1:HHGeDUpAsPa0pmOx7wrByCitjQ0mbUxf0R9v+g67uCA= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260707203317-661b54b51a33 h1:oW88YVT5ENU6rCPVOLV/hofmFBux2Mu1EOwz8KJu5ic= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260707203317-661b54b51a33/go.mod h1:/i8hjTPFdVWHiY+QjeSiVS2Z3GB3WAZznGgXHstC02E= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260804191526-b7a850ae7648 h1:WEUMkKQPAgcNMRgES6CBWrRUiII+HKEWQjulKQBSuMA= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260804191526-b7a850ae7648/go.mod h1:/i8hjTPFdVWHiY+QjeSiVS2Z3GB3WAZznGgXHstC02E= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b h1:QuI6SmQFK/zyUlVWEf0GMkiUYBPY4lssn26nKSd/bOM= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe h1:MDnY5wQbWTpFdDnMRicEnoMfSP5nM/KncARr4skP1ug= +github.com/smartcontractkit/chainlink-protos/metering/go v0.0.0-20260710151514-27b5a126dabe/go.mod h1:z7lx7wI3XZ4u9kmUtAVdwn1BCC9T8aieWSDcuDgPTdQ= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16 h1:/vkKPJoweLkRd56V4YHGRAtTG4+/JAlgklGEfvH6l4c= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260709145319-7782fb89eb16/go.mod h1:dkR2uYg9XYJuT1JASkPzWE51jjFkVb86P7a/yXe5/GM= github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260528173149-f5b8336b19d9 h1:LQy2j2+TdKLSWsUTUYuqmQPn8kjqCLjGI3ZJYGtDc08= @@ -226,11 +254,26 @@ github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 h1:12i github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7/go.mod h1:FX7/bVdoep147QQhsOPkYsPEXhGZjeYx6lBSaSXtZOA= github.com/smartcontractkit/libocr v0.0.0-20260810200708-618b5bf7f342 h1:pEcgcjTGA83MzpqbTbyIg9AJrOs62s77SooDdJGIg9w= github.com/smartcontractkit/libocr v0.0.0-20260810200708-618b5bf7f342/go.mod h1:5JPtsRwjugpyfsdEALC4RopfvohqK/G+3DHaR8uv+Bc= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= +github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E= +github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -238,8 +281,10 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.0 h1:K6Mr6jO9JICuend/5xzTM03ydSV3vdNRYAdPSukj8uI= +github.com/stretchr/testify v1.12.0/go.mod h1:bOYBZb5qJ00vPzWfIqBUZPaxK8jWiXc6d3ErP4Ca9Gw= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= @@ -307,11 +352,13 @@ go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= -golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20260508232706-74f9aab9d74a h1:+3jdDGGB8NGb1Zktc737jlt3/A5f6UlwSzmvqUuufxw= golang.org/x/exp v0.0.0-20260508232706-74f9aab9d74a/go.mod h1:d2fgXJLVs4dYDHUk5lwMIfzRzSrWCfGZb0ZqeLa/Vcw= @@ -321,8 +368,8 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= -golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -332,16 +379,16 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -357,10 +404,9 @@ golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/telemetry v0.0.0-20260519152614-eab6ae52b5e2 h1:2EucmYlcIsc8Y6aLj+kX90Y00hmjqLNlw935kc13R2k= golang.org/x/telemetry v0.0.0-20260519152614-eab6ae52b5e2/go.mod h1:Eqhaxk/wZsWEH8CRxLwj6xzEJbz7k1EFGqx7nyCoabE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -368,8 +414,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= -golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -380,8 +426,8 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= -golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -398,15 +444,15 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20210401141331-865547bb08e2/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5 h1:1VUiZAXyC+zmiFYi+WLtBzr68Cj8wOofHjjrA/kkizc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= -google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -418,8 +464,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/libs/standalone/protohelpers/ui/assets.go b/libs/standalone/protohelpers/ui/assets.go new file mode 100644 index 000000000..982c6e442 --- /dev/null +++ b/libs/standalone/protohelpers/ui/assets.go @@ -0,0 +1,95 @@ +package ui + +import ( + "crypto/sha256" + _ "embed" + "encoding/hex" + "encoding/json" + "fmt" +) + +// The page's own stylesheet and scripts. Both pages share the stylesheet, and the +// form page's script is served from grpcui's asset route while the fan-out page's +// is served from ours. +var ( + //go:embed resources/page.css + pageCSS string + + //go:embed resources/form.js + formJS string + + // The arithmetic for the special value types, shared by both pages: the + // encoding has to match Go's big.Int and decimal.Decimal exactly, and two + // copies of that would be two things to keep right. Prepended to each page's + // own script rather than served separately, so it cannot load second. + //go:embed resources/values.js + valuesJS string + + //go:embed resources/request.js + requestJS string + + // The subscription sidebar and its tables. Its own file rather than more of + // request.js: it is driven by what arrives on a stream rather than by the form, + // so the only thing the two share is the page they are on. + //go:embed resources/subscriptions.js + subscriptionsJS string + + //go:embed resources/request.html + requestHTML string +) + +// Assets are served under content-hashed names. +// +// grpcui serves what it is given with "Cache-Control: private, max-age=3600" and +// only revalidates its index, so at a fixed name a rebuilt binary would not reach +// an already-open browser for an hour - which looks exactly like the page being +// broken. A name that changes with the content cannot be served stale. +func hashedName(base, content, ext string) string { + sum := sha256.Sum256([]byte(content)) + return fmt.Sprintf("%s.%s.%s", base, hex.EncodeToString(sum[:])[:12], ext) +} + +func cssFileName() string { return hashedName("cre-debug", pageCSS, "css") } +func jsFileName(served string) string { return hashedName("cre-debug", served, "js") } +func requestJSFileName() string { + return hashedName("cre-debug-request", valuesJS+requestJS, "js") +} +func subscriptionsJSFileName() string { + return hashedName("cre-debug-subscriptions", subscriptionsJS, "js") +} + +// customJS is form.js with the page's configuration prepended, so the browser has +// it without a second request. +// +// The configuration is what keeps the page from guessing: the metadata fields come +// from the RequestMetadata type, and the methods from the descriptors the +// capabilities were generated against. +func customJS(s *Server) (string, error) { + cfg := struct { + Metadata []Field `json:"metadata"` + Prefix string `json:"headerPrefix"` + // Subscriptions are the services whose methods register a trigger, and + // TriggerIDHeader is what the trigger ID travels in. Both come from the + // descriptors rather than the page working out which is which from a name. + Subscriptions []string `json:"subscriptions"` + TriggerIDHeader string `json:"triggerIdHeader"` + // Path is where the pages are mounted, so the form can link to the fan-out + // page - which is where a subscription's events are shown. + Path string `json:"prefix"` + // Special are the messages the form offers a number for instead of their + // fields, and where a response holds them. See special.go. + Special SpecialConfig `json:"special"` + }{ + Metadata: Fields(), + Prefix: HeaderPrefix, + Subscriptions: s.subscriptionServices(), + TriggerIDHeader: TriggerIDHeader, + Path: s.prefix, + Special: s.specialConfig(), + } + encoded, err := json.Marshal(cfg) + if err != nil { + return "", fmt.Errorf("encoding the debug page config: %w", err) + } + return "window.__CRE_DEBUG__ = " + string(encoded) + ";\n" + valuesJS + "\n" + formJS, nil +} diff --git a/libs/standalone/protohelpers/ui/context.go b/libs/standalone/protohelpers/ui/context.go new file mode 100644 index 000000000..a9ab3a841 --- /dev/null +++ b/libs/standalone/protohelpers/ui/context.go @@ -0,0 +1,62 @@ +package ui + +import ( + "context" + "net/http" + + "google.golang.org/grpc/metadata" + + "github.com/smartcontractkit/chainlink-common/pkg/capabilities" +) + +// metadataFromContext reads the request metadata off the gRPC metadata the form +// generator forwarded from the browser's headers. +// +// The headers reach here as outgoing metadata, keyed lowercase, which is why the +// lookup lowercases rather than relying on the caller's casing. +func metadataFromContext(ctx context.Context) (capabilities.RequestMetadata, error) { + md, ok := metadata.FromOutgoingContext(ctx) + if !ok { + md = metadata.MD{} + } + return MetadataFromHeaders(func(name string) []string { + return md.Get(name) + }) +} + +// MetadataFromRequest is the same for an ordinary HTTP request, which is what the +// fan-out endpoint has: it holds headers rather than gRPC metadata. +func MetadataFromRequest(r *http.Request) (capabilities.RequestMetadata, error) { + return MetadataFromHeaders(func(name string) []string { + return r.Header.Values(name) + }) +} + +// triggerIDFromContext is the trigger ID a registration named, or a fresh one. +func triggerIDFromContext(ctx context.Context) string { + md, ok := metadata.FromOutgoingContext(ctx) + if !ok { + md = metadata.MD{} + } + return TriggerIDFromHeaders(md.Get) +} + +// HeaderNames is every header the metadata travels in, which is what the form +// generator has to be told to forward. +func HeaderNames() []string { + fields := Fields() + names := make([]string, 0, len(fields)) + for _, f := range fields { + names = append(names, f.Header) + } + return names +} + +// PreservedHeaders is every header a request may carry that has to reach Invoke: +// the metadata, and the trigger ID a subscription is identified by. +// +// The form generator drops anything it was not told to keep, so a header missing +// from here is one the page can offer a box for and never send. +func PreservedHeaders() []string { + return append(HeaderNames(), TriggerIDHeader) +} diff --git a/libs/standalone/protohelpers/ui/errors.go b/libs/standalone/protohelpers/ui/errors.go new file mode 100644 index 000000000..9391454c0 --- /dev/null +++ b/libs/standalone/protohelpers/ui/errors.go @@ -0,0 +1,102 @@ +package ui + +import ( + "errors" + "fmt" + "net/http" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + caperrors "github.com/smartcontractkit/chainlink-common/pkg/capabilities/errors" +) + +// The page tells the two kinds of failure apart, because they are not the same +// news: something the user typed is theirs to correct, and something this could +// not do is ours. +// +// The distinction is carried as a gRPC status, which is also what decides how a +// failure reaches the browser. grpcurl returns a plain error from an Invoke to +// grpcui as an infrastructure failure, which grpcui answers with a 500 - so a +// mistyped number would be reported as the server having broken. A status error +// instead comes back as a failed RPC, rendered in the Response tab where the user +// can see what they did and fix it. +// +// So: user errors are statuses, system errors are not, and 500 is reserved for the +// second kind. + +// userErrorf is a failure the caller can fix: a value that will not parse, a +// method that does not exist, a request that is not the shape the method takes. +func userErrorf(format string, args ...any) error { + return status.Errorf(codes.InvalidArgument, format, args...) +} + +// systemErrorf is a failure the caller cannot do anything about. Left as a plain +// error so it surfaces as one rather than as something the user got wrong. +func systemErrorf(format string, args ...any) error { + return fmt.Errorf(format, args...) +} + +// fromCapability classifies what a capability answered with. +// +// A capability that failed is not the page failing: it was asked something and it +// answered. So neither outcome is a 500 from this package's point of view - but a +// capability already says whose fault it was, via Origin, and repeating that is +// better than flattening it. OriginUser becomes a user error, so the browser is +// told what to change; anything else is reported as the capability's own failure. +func fromCapability(err error) error { + if err == nil { + return nil + } + + var capErr caperrors.Error + if errors.As(err, &capErr) { + if capErr.Origin() == caperrors.OriginUser { + return status.Error(codes.InvalidArgument, capErr.Error()) + } + return status.Error(codes.FailedPrecondition, capErr.Error()) + } + + // No origin to go on. Reported as a failed call rather than a broken page, + // because the call is what failed: the request reached a capability and came + // back with this. + return status.Error(codes.Unknown, err.Error()) +} + +// isUserError reports whether err is one of ours from userErrorf. +// +// Only InvalidArgument counts. A capability answering InvalidArgument of its own +// accord is saying the same thing about the same request, so treating it the same +// way is right rather than convenient. +func isUserError(err error) bool { + if err == nil { + return false + } + var se interface{ GRPCStatus() *status.Status } + if errors.As(err, &se) { + return se.GRPCStatus().Code() == codes.InvalidArgument + } + return false +} + +// httpStatus is the code a failure is answered with: the caller's mistake is a +// 400, and anything else is a 500 - which is what makes a 500 mean the page +// itself failed rather than that the request was wrong. +func httpStatus(err error) int { + if isUserError(err) { + return http.StatusBadRequest + } + return http.StatusInternalServerError +} + +// writeError answers a request with the failure and the code that fits it. +func writeError(w http.ResponseWriter, err error) { + message := err.Error() + if s, ok := status.FromError(err); ok { + // Without this the body reads "rpc error: code = InvalidArgument desc = + // ...", which is the transport talking rather than the thing that went + // wrong. + message = s.Message() + } + http.Error(w, message, httpStatus(err)) +} diff --git a/libs/standalone/protohelpers/ui/fleet.go b/libs/standalone/protohelpers/ui/fleet.go new file mode 100644 index 000000000..481baf972 --- /dev/null +++ b/libs/standalone/protohelpers/ui/fleet.go @@ -0,0 +1,99 @@ +package ui + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sync" +) + +// Instance is one instance's debug page, as the fan-out reaches it. +// +// It holds the instance's handler rather than its address. Instances of an embed +// run share a process, so a request to a sibling is a call into its handler: no +// port to work out, no socket, and nothing to go wrong between them. It also +// means the fan-out works when the gRPC servers were given port 0 and there is no +// arithmetic that could have found them. +type Instance struct { + Index int `json:"index"` + Label string `json:"label"` + + handler http.Handler +} + +// Fleet is every instance's page, shared by pointer between the configured +// dependency and the embedded form each instance resolves - the same way the +// embedded config is shared - so each instance adds itself to one list and any of +// them can reach the rest. +// +// Instances are constructed one after another and register during construction, +// so the list is complete before the process is serving. The mutex is for that +// construction, not for the requests that read it afterwards. +type Fleet struct { + mu sync.Mutex + instances []*Instance +} + +// Add registers an instance's page. Called once per instance, during construction. +func (f *Fleet) Add(in *Instance) { + f.mu.Lock() + defer f.mu.Unlock() + f.instances = append(f.instances, in) +} + +// List is every registered instance, in the order they were added, which is +// instance order. +func (f *Fleet) List() []*Instance { + f.mu.Lock() + defer f.mu.Unlock() + out := make([]*Instance, len(f.instances)) + copy(out, f.instances) + return out +} + +// invoke calls one method on this instance and returns what its page answered. +// +// The call goes through the instance's own handler, so it takes exactly the path a +// browser's request would: the same decoding of the posted JSON into the method's +// message, the same CSRF check, the same wrapping into a CapabilityRequest. Only +// the transport is skipped. +func (in *Instance) invoke(method string, body []byte, header http.Header) (json.RawMessage, error) { + // The index page is what issues the CSRF cookie, exactly as it would over a + // socket. Asking for it here keeps the handler's own check meaningful rather + // than working around it. + index := httptest.NewRecorder() + in.handler.ServeHTTP(index, httptest.NewRequest(http.MethodGet, "/", nil)) + + token := "" + for _, c := range index.Result().Cookies() { + if c.Name == csrfCookieName { + token = c.Value + break + } + } + if token == "" { + return nil, fmt.Errorf("instance %d did not issue a %s cookie", in.Index, csrfCookieName) + } + + req := httptest.NewRequest(http.MethodPost, "/invoke/"+method, bytes.NewReader(body)) + for name, values := range header { + for _, v := range values { + req.Header.Add(name, v) + } + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set(csrfHeaderName, token) + req.AddCookie(&http.Cookie{Name: csrfCookieName, Value: token}) + + rec := httptest.NewRecorder() + in.handler.ServeHTTP(rec, req) + + result := rec.Result() + defer result.Body.Close() + if result.StatusCode != http.StatusOK { + return nil, fmt.Errorf("instance %d returned %d: %s", in.Index, result.StatusCode, bytes.TrimSpace(rec.Body.Bytes())) + } + return json.RawMessage(rec.Body.Bytes()), nil +} diff --git a/libs/standalone/protohelpers/ui/hub.go b/libs/standalone/protohelpers/ui/hub.go new file mode 100644 index 000000000..ae0fb335b --- /dev/null +++ b/libs/standalone/protohelpers/ui/hub.go @@ -0,0 +1,602 @@ +package ui + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "sort" + "sync" + "time" + + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/smartcontractkit/chainlink-common/pkg/capabilities" +) + +// A trigger is not a call, so it does not fit the request-and-response the rest +// of the page is: it is registered once and then delivers whatever it delivers, +// for as long as it is registered, to however many instances registered it. +// +// The Hub is what holds that. A subscription is keyed by its trigger ID, which is +// the identifier the registration carried, so registering the same trigger ID on +// another instance later joins the subscription already running rather than +// starting a second one - which is what makes "send to two instances now, a third +// in a minute" one table rather than two. +// +// Within a subscription an event is keyed by its own ID, and every instance that +// delivered it is a column. Instances are meant to agree; when they do not, that +// is the thing worth seeing, so the payloads are hashed and a row that carries +// more than one hash is marked rather than averaged away. + +const ( + // DefaultGrace is how long a subscription outlives the last reader watching + // it. Closing the window is how a subscription is closed, and a reload closes + // the window - so the two are told apart by waiting to see whether anyone + // comes back. + DefaultGrace = time.Minute + + // DefaultRing is how many events a subscription keeps. A reader that + // reattaches is sent them, so the table it left is the table it returns to. + DefaultRing = 200 + + // clientBuffer is how far behind a reader may fall before it is dropped. + // Dropping is safe: the browser reconnects and is sent the whole table, which + // is more correct than a reader that has silently missed a row. + clientBuffer = 64 +) + +// Hub holds every live subscription of the process, keyed by trigger ID. +// +// One Hub is shared by every instance, the same way the Fleet is: an embed run's +// instances are separate registries but one browser, so a subscription registered +// across four of them has to be one thing for the page to show it as one table. +type Hub struct { + grace time.Duration + ring int + + // ctx outlives the request that registered a trigger. A registration is not + // the request's to own: the request returns as soon as the trigger is + // registered, and the events arrive long afterwards. + ctx context.Context + cancel context.CancelFunc + + mu sync.Mutex + subs map[string]*subscription +} + +// NewHub builds an empty Hub with the default grace and ring. +func NewHub() *Hub { + ctx, cancel := context.WithCancel(context.Background()) + return &Hub{ + grace: DefaultGrace, + ring: DefaultRing, + ctx: ctx, + cancel: cancel, + subs: map[string]*subscription{}, + } +} + +// Close unregisters every subscription and stops reading them. +func (h *Hub) Close() error { + h.mu.Lock() + subs := make([]*subscription, 0, len(h.subs)) + for _, s := range h.subs { + subs = append(subs, s) + } + h.subs = map[string]*subscription{} + h.mu.Unlock() + + var errs []error + for _, s := range subs { + errs = append(errs, s.close()) + } + h.cancel() + return errors.Join(errs...) +} + +// registration is one instance joining one subscription. +type registration struct { + triggerID string + capabilityID string + // service is the real service the trigger belongs to, for the page to show. + service string + method string + + instance int + label string + trigger capabilities.TriggerExecutable + + metadata capabilities.RequestMetadata + payload *anypb.Any + + // eventType is the generated Go type of the trigger's streamed message, which + // is how a delivered event is read out of the Any it arrives in. + eventType reflect.Type +} + +// subscribe registers one instance's trigger, joining the subscription with this +// trigger ID or starting it. +func (h *Hub) subscribe(r registration) (*Status, error) { + s, err := h.subscription(r) + if err != nil { + return nil, err + } + + if err = s.attach(r); err != nil { + // A subscription nobody managed to attach to is not left behind: it would + // sit in the sidebar claiming to be watching something. + h.dropIfEmpty(s) + return nil, err + } + + status := s.status(false) + return &status, nil +} + +// subscription finds the one this registration belongs to, or starts it. +// +// A trigger ID names a subscription, so one that already exists has to be the +// same trigger: joining "the cron I started" with a different method would put +// two unrelated streams in one table. +func (h *Hub) subscription(r registration) (*subscription, error) { + h.mu.Lock() + defer h.mu.Unlock() + + if existing, ok := h.subs[r.triggerID]; ok { + if existing.capabilityID != r.capabilityID || existing.method != r.method { + return nil, userErrorf( + "trigger ID %s is already subscribed to %s/%s, so it cannot also subscribe to %s/%s - use a different trigger ID", + r.triggerID, existing.capabilityID, existing.method, r.capabilityID, r.method) + } + return existing, nil + } + + s := &subscription{ + hub: h, + triggerID: r.triggerID, + capabilityID: r.capabilityID, + service: r.service, + method: r.method, + eventType: r.eventType, + created: time.Now().UTC(), + attached: map[int]*attachment{}, + events: map[string]*Row{}, + clients: map[*client]struct{}{}, + } + // The grace clock starts now, not when the first reader leaves: a subscription + // nobody ever watches is one nobody is going to close. + s.startGrace() + + h.subs[r.triggerID] = s + return s, nil +} + +// dropIfEmpty removes a subscription nothing is attached to. +func (h *Hub) dropIfEmpty(s *subscription) { + s.mu.Lock() + empty := len(s.attached) == 0 + s.mu.Unlock() + if !empty { + return + } + + h.mu.Lock() + defer h.mu.Unlock() + if h.subs[s.triggerID] == s { + delete(h.subs, s.triggerID) + } +} + +// get is the subscription with this trigger ID. +func (h *Hub) get(triggerID string) (*subscription, error) { + h.mu.Lock() + defer h.mu.Unlock() + s, ok := h.subs[triggerID] + if !ok { + return nil, userErrorf("no subscription with trigger ID %s", triggerID) + } + return s, nil +} + +// List is every live subscription, newest last, for the page's sidebar. +func (h *Hub) List() []Status { + h.mu.Lock() + subs := make([]*subscription, 0, len(h.subs)) + for _, s := range h.subs { + subs = append(subs, s) + } + h.mu.Unlock() + + sort.Slice(subs, func(i, j int) bool { return subs[i].created.Before(subs[j].created) }) + + out := make([]Status, 0, len(subs)) + for _, s := range subs { + out = append(out, s.status(false)) + } + return out +} + +// Unsubscribe closes a subscription, or detaches the instances named. +func (h *Hub) Unsubscribe(triggerID string, instances []int) error { + s, err := h.get(triggerID) + if err != nil { + return err + } + + if len(instances) == 0 { + h.remove(s) + return s.close() + } + + var errs []error + for _, index := range instances { + errs = append(errs, s.detach(index)) + } + h.dropIfEmpty(s) + return errors.Join(errs...) +} + +// Ack forwards a reader's acknowledgement to every instance that delivered the +// event. +// +// The page acks rather than this doing it on delivery: a capability that redelivers +// what was not acknowledged is doing what it is meant to, and acknowledging an +// event the browser has not been shown would hide exactly the delivery being +// debugged. +func (h *Hub) Ack(triggerID, eventID string) error { + s, err := h.get(triggerID) + if err != nil { + return err + } + return s.ack(h.ctx, eventID) +} + +// remove takes a subscription out of the hub without closing it. +func (h *Hub) remove(s *subscription) { + h.mu.Lock() + defer h.mu.Unlock() + if h.subs[s.triggerID] == s { + delete(h.subs, s.triggerID) + } +} + +// subscription is one trigger ID: the instances registered under it, the events +// they delivered, and the readers watching. +type subscription struct { + hub *Hub + + triggerID string + capabilityID string + service string + method string + eventType reflect.Type + created time.Time + + mu sync.Mutex + attached map[int]*attachment + events map[string]*Row + order []string + clients map[*client]struct{} + graceTimer *time.Timer + closed bool +} + +// attachment is one instance's registration, kept because unregistering takes the +// request that registered. +type attachment struct { + instance int + label string + trigger capabilities.TriggerExecutable + request capabilities.TriggerRegistrationRequest + stop context.CancelFunc +} + +// attach registers the trigger on one instance and starts reading its events. +func (s *subscription) attach(r registration) error { + request := capabilities.TriggerRegistrationRequest{ + TriggerID: r.triggerID, + Metadata: r.metadata, + Method: r.method, + Payload: r.payload, + } + + s.mu.Lock() + if s.closed { + s.mu.Unlock() + return userErrorf("subscription %s has been closed", s.triggerID) + } + if _, taken := s.attached[r.instance]; taken { + s.mu.Unlock() + return userErrorf("%s is already subscribed to trigger ID %s", r.label, s.triggerID) + } + s.mu.Unlock() + + // Registered outside the lock: this reaches the capability, and holding the + // subscription's lock across it would stall every reader of every event while + // one instance registers. + ctx, stop := context.WithCancel(s.hub.ctx) + events, err := r.trigger.RegisterTrigger(ctx, request) + if err != nil { + stop() + return fromCapability(err) + } + + a := &attachment{instance: r.instance, label: r.label, trigger: r.trigger, request: request, stop: stop} + + s.mu.Lock() + if s.closed { + s.mu.Unlock() + stop() + return userErrorf("subscription %s has been closed", s.triggerID) + } + if _, taken := s.attached[r.instance]; taken { + // Lost a race with another registration of the same instance. Undone rather + // than left registered twice. + s.mu.Unlock() + stop() + _ = r.trigger.UnregisterTrigger(s.hub.ctx, request) + return userErrorf("%s is already subscribed to trigger ID %s", r.label, s.triggerID) + } + s.attached[r.instance] = a + s.mu.Unlock() + + go s.read(ctx, a, events) + s.broadcast(Message{Type: MessageAttached, TriggerID: s.triggerID, Status: ptr(s.status(false))}) + return nil +} + +// detach unregisters one instance and stops reading it. +func (s *subscription) detach(instance int) error { + s.mu.Lock() + a, ok := s.attached[instance] + if ok { + delete(s.attached, instance) + } + s.mu.Unlock() + + if !ok { + return userErrorf("instance %d is not subscribed to trigger ID %s", instance, s.triggerID) + } + + a.stop() + err := a.trigger.UnregisterTrigger(s.hub.ctx, a.request) + s.broadcast(Message{Type: MessageAttached, TriggerID: s.triggerID, Status: ptr(s.status(false))}) + if err != nil { + return fromCapability(err) + } + return nil +} + +// close unregisters every instance and tells every reader the subscription is over. +func (s *subscription) close() error { + s.mu.Lock() + if s.closed { + s.mu.Unlock() + return nil + } + s.closed = true + if s.graceTimer != nil { + s.graceTimer.Stop() + s.graceTimer = nil + } + attached := make([]*attachment, 0, len(s.attached)) + for _, a := range s.attached { + attached = append(attached, a) + } + s.attached = map[int]*attachment{} + s.mu.Unlock() + + var errs []error + for _, a := range attached { + a.stop() + if err := a.trigger.UnregisterTrigger(s.hub.ctx, a.request); err != nil { + errs = append(errs, fmt.Errorf("%s: %w", a.label, err)) + } + } + + s.broadcast(Message{Type: MessageClosed, TriggerID: s.triggerID, Status: ptr(s.status(true))}) + s.disconnectAll() + return errors.Join(errs...) +} + +// read forwards one instance's events until the channel closes or the attachment +// is stopped. +func (s *subscription) read(ctx context.Context, a *attachment, events <-chan capabilities.TriggerResponse) { + for { + select { + case <-ctx.Done(): + return + case response, open := <-events: + if !open { + return + } + s.record(a, response) + } + } +} + +// record merges one instance's delivery of one event into its row. +func (s *subscription) record(a *attachment, response capabilities.TriggerResponse) { + node := Node{ + Instance: a.instance, + Label: a.label, + At: time.Now().UTC(), + } + if response.Err != nil { + node.Error = response.Err.Error() + } + + payload, id, err := s.decode(response.Event) + if err != nil && node.Error == "" { + node.Error = err.Error() + } + if err == nil { + node.PayloadID = id + } + + // An event with no ID cannot be a row of its own without every delivery + // looking like a separate event, so it is keyed by what it carried instead. + key := response.Event.ID + if key == "" { + key = "(no event ID) " + id + } + + row := s.merge(key, node, payload) + if row != nil { + s.broadcast(Message{Type: MessageRow, TriggerID: s.triggerID, Row: row}) + } +} + +// decode reads a delivered event out of the Any it arrives in, as the generated Go +// type the trigger declares, and hashes it. +// +// The hash is what makes disagreement visible: identical payloads hash the same, +// so a row with one hash is every instance agreeing and a row with two is not. +// Marshalling is deterministic so that a map field cannot make two equal payloads +// look different. +func (s *subscription) decode(event capabilities.TriggerEvent) (json.RawMessage, string, error) { + if event.Payload == nil { + if event.Outputs != nil { + // The values.Map path, which only a DAG registration takes. This + // package always registers with a payload, so seeing one means the + // capability answered a request it was not sent. + return nil, "", fmt.Errorf("the event carried Outputs rather than a Payload") + } + return nil, "", fmt.Errorf("the event carried no payload") + } + + message, ok := reflect.New(s.eventType.Elem()).Interface().(proto.Message) + if !ok { + return nil, "", fmt.Errorf("%s is not a protobuf message", s.eventType) + } + if err := event.Payload.UnmarshalTo(message); err != nil { + return nil, "", fmt.Errorf("failed to read the event as %T: %w", message, err) + } + + // Proto names, which is the spelling the response tab already uses, so an + // event and a response read the same way. + encoded, err := protojson.MarshalOptions{UseProtoNames: true}.Marshal(message) + if err != nil { + return nil, "", fmt.Errorf("failed to encode the event: %w", err) + } + + canonical, err := proto.MarshalOptions{Deterministic: true}.Marshal(message) + if err != nil { + return nil, "", fmt.Errorf("failed to hash the event: %w", err) + } + return encoded, shortHash(canonical), nil +} + +// merge folds a node into its row and returns the row to send, or nil if the +// subscription is closed. +func (s *subscription) merge(id string, node Node, payload json.RawMessage) *Row { + s.mu.Lock() + defer s.mu.Unlock() + + if s.closed { + return nil + } + + row, ok := s.events[id] + if !ok { + row = &Row{ID: id, First: node.At} + s.events[id] = row + s.order = append(s.order, id) + s.trim() + } + + row.add(node, payload) + + copied := row.clone() + return &copied +} + +// trim keeps the kept-event count at the ring size. Called with the lock held. +func (s *subscription) trim() { + for len(s.order) > s.hub.ring { + delete(s.events, s.order[0]) + s.order = s.order[1:] + } +} + +// rows is every kept event, oldest first. Copied, so a reader is never handed +// something another delivery is about to change under it. +func (s *subscription) rows() []Row { + s.mu.Lock() + defer s.mu.Unlock() + + out := make([]Row, 0, len(s.order)) + for _, id := range s.order { + if row, ok := s.events[id]; ok { + out = append(out, row.clone()) + } + } + return out +} + +// status describes the subscription, with its events when withRows. +func (s *subscription) status(closed bool) Status { + s.mu.Lock() + instances := make([]Attached, 0, len(s.attached)) + for _, a := range s.attached { + instances = append(instances, Attached{Instance: a.instance, Label: a.label}) + } + events := len(s.order) + readers := len(s.clients) + graced := s.graceTimer != nil + if s.closed { + closed = true + } + s.mu.Unlock() + + sort.Slice(instances, func(i, j int) bool { return instances[i].Instance < instances[j].Instance }) + + return Status{ + TriggerID: s.triggerID, + CapabilityID: s.capabilityID, + Service: s.service, + Method: s.method, + Instances: instances, + Events: events, + Readers: readers, + InGrace: graced && readers == 0, + Closed: closed, + Created: s.created, + } +} + +// ack forwards an acknowledgement to every instance that delivered the event. +func (s *subscription) ack(ctx context.Context, eventID string) error { + s.mu.Lock() + row, ok := s.events[eventID] + var targets []*attachment + if ok { + for _, node := range row.Nodes { + if a, attached := s.attached[node.Instance]; attached { + targets = append(targets, a) + } + } + } + s.mu.Unlock() + + if !ok { + return userErrorf("trigger ID %s has no event %s", s.triggerID, eventID) + } + + var errs []error + for _, a := range targets { + if err := a.trigger.AckEvent(ctx, s.triggerID, eventID, s.method); err != nil { + errs = append(errs, fmt.Errorf("%s: %w", a.label, err)) + } + } + if err := errors.Join(errs...); err != nil { + return fromCapability(err) + } + return nil +} + +func ptr[T any](v T) *T { return &v } diff --git a/libs/standalone/protohelpers/ui/hub_test.go b/libs/standalone/protohelpers/ui/hub_test.go new file mode 100644 index 000000000..dcf18cd3f --- /dev/null +++ b/libs/standalone/protohelpers/ui/hub_test.go @@ -0,0 +1,574 @@ +package ui + +import ( + "context" + "fmt" + "reflect" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/smartcontractkit/chainlink-common/pkg/capabilities" + "github.com/smartcontractkit/chainlink-common/pkg/capabilities/pb" +) + +// The trigger the tests subscribe to is Executable.Execute, which is the +// streaming method of the same real service the rest of these tests use. So its +// event is a CapabilityResponse, whose Error field is a convenient thing to make +// two instances disagree about. +func eventType() reflect.Type { return reflect.TypeFor[*pb.CapabilityResponse]() } + +func event(t *testing.T, id, payload string) capabilities.TriggerResponse { + t.Helper() + wrapped, err := anypb.New(&pb.CapabilityResponse{Error: payload}) + require.NoError(t, err) + return capabilities.TriggerResponse{Event: capabilities.TriggerEvent{ID: id, Payload: wrapped}} +} + +// fakeTrigger is one instance's trigger capability, whose events the test +// delivers by hand. +// +// A channel per registration rather than one for the capability: the point of +// most of these tests is several instances delivering the same event, which is +// several registrations. +type fakeTrigger struct { + registerErr error + + mu sync.Mutex + channels []chan capabilities.TriggerResponse + registered []capabilities.TriggerRegistrationRequest + unregistered []capabilities.TriggerRegistrationRequest + acked []string +} + +func (f *fakeTrigger) Info(context.Context) (capabilities.CapabilityInfo, error) { + return capabilities.NewCapabilityInfo(testCapabilityID, capabilities.CapabilityTypeTrigger, "a trigger for tests") +} + +func (f *fakeTrigger) RegisterTrigger(_ context.Context, request capabilities.TriggerRegistrationRequest) (<-chan capabilities.TriggerResponse, error) { + if f.registerErr != nil { + return nil, f.registerErr + } + + f.mu.Lock() + defer f.mu.Unlock() + ch := make(chan capabilities.TriggerResponse, 16) + f.channels = append(f.channels, ch) + f.registered = append(f.registered, request) + return ch, nil +} + +func (f *fakeTrigger) UnregisterTrigger(_ context.Context, request capabilities.TriggerRegistrationRequest) error { + f.mu.Lock() + defer f.mu.Unlock() + f.unregistered = append(f.unregistered, request) + return nil +} + +func (f *fakeTrigger) AckEvent(_ context.Context, triggerID, eventID, method string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.acked = append(f.acked, fmt.Sprintf("%s/%s/%s", triggerID, eventID, method)) + return nil +} + +// deliver sends an event down the nth registration this capability handed out. +func (f *fakeTrigger) deliver(t *testing.T, n int, response capabilities.TriggerResponse) { + t.Helper() + f.mu.Lock() + require.Greater(t, len(f.channels), n, "registration %d was never made", n) + ch := f.channels[n] + f.mu.Unlock() + ch <- response +} + +func (f *fakeTrigger) requests() []capabilities.TriggerRegistrationRequest { + f.mu.Lock() + defer f.mu.Unlock() + return append([]capabilities.TriggerRegistrationRequest(nil), f.registered...) +} + +func (f *fakeTrigger) unregisters() []capabilities.TriggerRegistrationRequest { + f.mu.Lock() + defer f.mu.Unlock() + return append([]capabilities.TriggerRegistrationRequest(nil), f.unregistered...) +} + +func (f *fakeTrigger) acks() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.acked...) +} + +// join subscribes one instance to a trigger ID. +func join(t *testing.T, h *Hub, triggerID string, instance int, trigger capabilities.TriggerExecutable) (*Status, error) { + t.Helper() + payload, err := anypb.New(&pb.CapabilityRequest{}) + require.NoError(t, err) + + return h.subscribe(registration{ + triggerID: triggerID, + capabilityID: testCapabilityID, + service: string(testService().FullName()), + method: "Execute", + instance: instance, + label: fmt.Sprintf("instance %d", instance+1), + trigger: trigger, + payload: payload, + eventType: eventType(), + }) +} + +// eventually waits for a condition the hub reaches on a goroutine of its own. +func eventually(t *testing.T, why string, condition func() bool) { + t.Helper() + require.Eventually(t, condition, 2*time.Second, time.Millisecond, why) +} + +// A subscription registers the trigger with the ID and method it was asked for. +func TestSubscribeRegistersTheTrigger(t *testing.T) { + h := NewHub() + t.Cleanup(func() { _ = h.Close() }) + + trigger := &fakeTrigger{} + status, err := join(t, h, "ui-trigger-abc", 0, trigger) + require.NoError(t, err) + + assert.Equal(t, "ui-trigger-abc", status.TriggerID) + assert.Equal(t, "Execute", status.Method) + assert.Equal(t, testCapabilityID, status.CapabilityID) + require.Len(t, status.Instances, 1) + assert.Equal(t, "instance 1", status.Instances[0].Label) + + requests := trigger.requests() + require.Len(t, requests, 1) + assert.Equal(t, "ui-trigger-abc", requests[0].TriggerID) + assert.Equal(t, "Execute", requests[0].Method) + assert.NotNil(t, requests[0].Payload, "the form's input is what the trigger is configured with") +} + +// The trigger ID is the subscription, so a second instance registering the same +// one joins it rather than starting another. This is what makes "two instances +// now, a third later" one table. +func TestSubscribingTheSameTriggerIDJoins(t *testing.T) { + h := NewHub() + t.Cleanup(func() { _ = h.Close() }) + + first, second := &fakeTrigger{}, &fakeTrigger{} + _, err := join(t, h, "ui-trigger-shared", 0, first) + require.NoError(t, err) + + status, err := join(t, h, "ui-trigger-shared", 3, second) + require.NoError(t, err) + + require.Len(t, status.Instances, 2) + assert.Equal(t, 0, status.Instances[0].Instance) + assert.Equal(t, 3, status.Instances[1].Instance) + assert.Len(t, h.List(), 1, "one subscription, not one per instance") +} + +// A trigger ID already watching something else would put two unrelated streams in +// one table, so it is refused - and refused as the caller's mistake, since a +// different ID fixes it. +func TestSubscribingTheSameTriggerIDToAnotherMethodIsRefused(t *testing.T) { + h := NewHub() + t.Cleanup(func() { _ = h.Close() }) + + _, err := join(t, h, "ui-trigger-taken", 0, &fakeTrigger{}) + require.NoError(t, err) + + payload, err := anypb.New(&pb.CapabilityRequest{}) + require.NoError(t, err) + _, err = h.subscribe(registration{ + triggerID: "ui-trigger-taken", + capabilityID: testCapabilityID, + method: "SomethingElse", + instance: 1, + trigger: &fakeTrigger{}, + payload: payload, + eventType: eventType(), + }) + require.Error(t, err) + assert.True(t, isUserError(err), "%v", err) + assert.Contains(t, err.Error(), "already subscribed") +} + +// Registering one instance twice under one trigger ID would register it twice in +// the capability, so it is refused rather than done. +func TestSubscribingOneInstanceTwiceIsRefused(t *testing.T) { + h := NewHub() + t.Cleanup(func() { _ = h.Close() }) + + trigger := &fakeTrigger{} + _, err := join(t, h, "ui-trigger-dup", 2, trigger) + require.NoError(t, err) + + _, err = join(t, h, "ui-trigger-dup", 2, trigger) + require.Error(t, err) + assert.True(t, isUserError(err), "%v", err) + assert.Len(t, trigger.requests(), 1, "the second attempt must not reach the capability") +} + +// A registration the capability refuses is not left behind as a subscription +// nothing is attached to. +func TestFailedRegistrationLeavesNoSubscription(t *testing.T) { + h := NewHub() + t.Cleanup(func() { _ = h.Close() }) + + _, err := join(t, h, "ui-trigger-bad", 0, &fakeTrigger{registerErr: fmt.Errorf("no such schedule")}) + require.Error(t, err) + assert.Empty(t, h.List()) +} + +// Two instances delivering the same event is one row with a column each, and one +// payload hash: they agree. +func TestAgreeingInstancesAreOneRow(t *testing.T) { + h := NewHub() + t.Cleanup(func() { _ = h.Close() }) + + first, second := &fakeTrigger{}, &fakeTrigger{} + _, err := join(t, h, "ui-trigger-agree", 0, first) + require.NoError(t, err) + _, err = join(t, h, "ui-trigger-agree", 1, second) + require.NoError(t, err) + + s, err := h.get("ui-trigger-agree") + require.NoError(t, err) + + first.deliver(t, 0, event(t, "event-1", "same")) + second.deliver(t, 0, event(t, "event-1", "same")) + + eventually(t, "both instances should appear in the row", func() bool { + rows := s.rows() + return len(rows) == 1 && len(rows[0].Nodes) == 2 + }) + + rows := s.rows() + require.Len(t, rows, 1) + assert.Equal(t, "event-1", rows[0].ID) + assert.Len(t, rows[0].PayloadIDs, 1, "identical payloads hash the same") + assert.False(t, rows[0].Diverged) + + // One payload, held once on the row rather than repeated per instance, and both + // nodes point at it. + require.Len(t, rows[0].Payloads, 1) + assert.JSONEq(t, `{"error":"same"}`, string(rows[0].Payloads[0])) + assert.Equal(t, 0, rows[0].Nodes[0].PayloadIndex) + assert.Equal(t, 0, rows[0].Nodes[1].PayloadIndex) +} + +// The payloads and their hashes are in the same order, and a node says which of +// them it sent - which is what lets the table put a payload per column and have +// the instance row point into it. +func TestPayloadsAndHashesLineUp(t *testing.T) { + h := NewHub() + t.Cleanup(func() { _ = h.Close() }) + + first, second, third := &fakeTrigger{}, &fakeTrigger{}, &fakeTrigger{} + for i, trigger := range []*fakeTrigger{first, second, third} { + _, err := join(t, h, "ui-trigger-lineup", i, trigger) + require.NoError(t, err) + } + + s, err := h.get("ui-trigger-lineup") + require.NoError(t, err) + + // Instances 1 and 3 agree, instance 2 does not. Delivered in instance order so + // the arrival order is the one the row is expected to keep. + first.deliver(t, 0, event(t, "event-1", "agreed")) + eventually(t, "the first instance should be recorded", func() bool { + return len(s.rows()) == 1 && len(s.rows()[0].Nodes) == 1 + }) + second.deliver(t, 0, event(t, "event-1", "different")) + eventually(t, "the second instance should be recorded", func() bool { + return len(s.rows()[0].Nodes) == 2 + }) + third.deliver(t, 0, event(t, "event-1", "agreed")) + eventually(t, "the third instance should be recorded", func() bool { + return len(s.rows()[0].Nodes) == 3 + }) + + row := s.rows()[0] + assert.True(t, row.Diverged) + + // Two distinct payloads, in the order they arrived, and as many hashes as + // payloads. + require.Len(t, row.Payloads, 2) + require.Len(t, row.PayloadIDs, 2) + assert.JSONEq(t, `{"error":"agreed"}`, string(row.Payloads[0])) + assert.JSONEq(t, `{"error":"different"}`, string(row.Payloads[1])) + + // And each instance points at the one it sent. + assert.Equal(t, 0, row.Nodes[0].PayloadIndex) + assert.Equal(t, 1, row.Nodes[1].PayloadIndex) + assert.Equal(t, 0, row.Nodes[2].PayloadIndex, "the third instance agreed with the first") + + // The hash at an index is the hash of the payload at that index. + for i, node := range row.Nodes { + assert.Equal(t, row.PayloadIDs[node.PayloadIndex], node.PayloadID, "node %d", i) + } +} + +// An instance that failed has no payload to point at, so it points at nothing +// rather than at payload zero - which would read as agreeing with it. +func TestAFailedDeliveryPointsAtNoPayload(t *testing.T) { + h := NewHub() + t.Cleanup(func() { _ = h.Close() }) + + trigger := &fakeTrigger{} + _, err := join(t, h, "ui-trigger-failed", 0, trigger) + require.NoError(t, err) + + s, err := h.get("ui-trigger-failed") + require.NoError(t, err) + + trigger.deliver(t, 0, capabilities.TriggerResponse{ + Event: capabilities.TriggerEvent{ID: "event-1"}, + Err: fmt.Errorf("the schedule was withdrawn"), + }) + eventually(t, "the failure should be recorded", func() bool { return len(s.rows()) == 1 }) + + row := s.rows()[0] + require.Len(t, row.Nodes, 1) + assert.Equal(t, -1, row.Nodes[0].PayloadIndex) + assert.Contains(t, row.Nodes[0].Error, "the schedule was withdrawn") + assert.Empty(t, row.Payloads) + assert.False(t, row.Diverged, "one instance failing is not two instances disagreeing") +} + +// Instances disagreeing is the bug the table exists to show, so the row says so +// rather than showing one of the two. +func TestDisagreeingInstancesDiverge(t *testing.T) { + h := NewHub() + t.Cleanup(func() { _ = h.Close() }) + + first, second := &fakeTrigger{}, &fakeTrigger{} + _, err := join(t, h, "ui-trigger-differ", 0, first) + require.NoError(t, err) + _, err = join(t, h, "ui-trigger-differ", 1, second) + require.NoError(t, err) + + s, err := h.get("ui-trigger-differ") + require.NoError(t, err) + + first.deliver(t, 0, event(t, "event-1", "this")) + second.deliver(t, 0, event(t, "event-1", "that")) + + eventually(t, "both instances should appear in the row", func() bool { + rows := s.rows() + return len(rows) == 1 && len(rows[0].Nodes) == 2 + }) + + rows := s.rows() + assert.True(t, rows[0].Diverged) + assert.Len(t, rows[0].PayloadIDs, 2) + assert.NotEqual(t, rows[0].Nodes[0].PayloadID, rows[0].Nodes[1].PayloadID) +} + +// The kept-event count is bounded, so a trigger firing all day does not grow +// without limit. +func TestEventsAreTrimmedToTheRing(t *testing.T) { + h := NewHub() + h.ring = 3 + t.Cleanup(func() { _ = h.Close() }) + + trigger := &fakeTrigger{} + _, err := join(t, h, "ui-trigger-ring", 0, trigger) + require.NoError(t, err) + + s, err := h.get("ui-trigger-ring") + require.NoError(t, err) + + for i := range 6 { + trigger.deliver(t, 0, event(t, fmt.Sprintf("event-%d", i), "payload")) + } + + eventually(t, "the oldest events should be dropped", func() bool { + rows := s.rows() + return len(rows) == 3 && rows[0].ID == "event-3" + }) +} + +// Nobody watching means nobody wants it, once the grace period has passed: this +// is what closing the window does. +func TestAbandonedSubscriptionsAreUnregistered(t *testing.T) { + h := NewHub() + h.grace = 10 * time.Millisecond + t.Cleanup(func() { _ = h.Close() }) + + trigger := &fakeTrigger{} + _, err := join(t, h, "ui-trigger-abandoned", 0, trigger) + require.NoError(t, err) + + eventually(t, "the trigger should be unregistered", func() bool { + return len(trigger.unregisters()) == 1 + }) + assert.Empty(t, h.List()) +} + +// A reader arriving inside the grace period is somebody coming back, which is +// what a reload looks like - so the subscription is still there. +func TestAReaderCancelsTheGracePeriod(t *testing.T) { + h := NewHub() + h.grace = 50 * time.Millisecond + t.Cleanup(func() { _ = h.Close() }) + + trigger := &fakeTrigger{} + _, err := join(t, h, "ui-trigger-kept", 0, trigger) + require.NoError(t, err) + + s, err := h.get("ui-trigger-kept") + require.NoError(t, err) + + c := newClient() + s.addClient(c) + + time.Sleep(150 * time.Millisecond) + assert.Empty(t, trigger.unregisters(), "a watched subscription must not be unregistered") + assert.Len(t, h.List(), 1) + + // And leaving starts the clock again. + s.removeClient(c) + eventually(t, "the trigger should be unregistered once the reader leaves", func() bool { + return len(trigger.unregisters()) == 1 + }) +} + +// A reader that reattaches is sent the table it left, which is what makes a +// reload look like nothing happened. +func TestASnapshotCarriesTheTable(t *testing.T) { + h := NewHub() + t.Cleanup(func() { _ = h.Close() }) + + trigger := &fakeTrigger{} + _, err := join(t, h, "ui-trigger-snapshot", 0, trigger) + require.NoError(t, err) + + s, err := h.get("ui-trigger-snapshot") + require.NoError(t, err) + + trigger.deliver(t, 0, event(t, "event-1", "first")) + trigger.deliver(t, 0, event(t, "event-2", "second")) + eventually(t, "both events should be recorded", func() bool { return len(s.rows()) == 2 }) + + snapshot := s.addClient(newClient()) + require.Len(t, snapshot.Rows, 2) + assert.Equal(t, "event-1", snapshot.Rows[0].ID) + assert.Equal(t, 1, snapshot.Readers) +} + +// A reader watching is sent each event as it arrives. +func TestReadersAreSentRows(t *testing.T) { + h := NewHub() + t.Cleanup(func() { _ = h.Close() }) + + trigger := &fakeTrigger{} + _, err := join(t, h, "ui-trigger-live", 0, trigger) + require.NoError(t, err) + + s, err := h.get("ui-trigger-live") + require.NoError(t, err) + + c := newClient() + s.addClient(c) + + trigger.deliver(t, 0, event(t, "event-1", "hello")) + + select { + case m := <-c.messages: + assert.Equal(t, MessageRow, m.Type) + require.NotNil(t, m.Row) + assert.Equal(t, "event-1", m.Row.ID) + case <-time.After(2 * time.Second): + t.Fatal("the reader was sent nothing") + } +} + +// Acknowledging goes to the instances that delivered the event, and carries the +// method, because that is what a capability keys its delivery on. +func TestAckReachesTheInstancesThatDelivered(t *testing.T) { + h := NewHub() + t.Cleanup(func() { _ = h.Close() }) + + first, second := &fakeTrigger{}, &fakeTrigger{} + _, err := join(t, h, "ui-trigger-ack", 0, first) + require.NoError(t, err) + _, err = join(t, h, "ui-trigger-ack", 1, second) + require.NoError(t, err) + + s, err := h.get("ui-trigger-ack") + require.NoError(t, err) + + // Only the first instance delivers it. + first.deliver(t, 0, event(t, "event-1", "hello")) + eventually(t, "the event should be recorded", func() bool { return len(s.rows()) == 1 }) + + require.NoError(t, h.Ack("ui-trigger-ack", "event-1")) + assert.Equal(t, []string{"ui-trigger-ack/event-1/Execute"}, first.acks()) + assert.Empty(t, second.acks(), "an instance that never delivered it has nothing to acknowledge") +} + +// Acking something that was never delivered is the caller's mistake, not a broken +// page. +func TestAckOfAnUnknownEventIsAUserError(t *testing.T) { + h := NewHub() + t.Cleanup(func() { _ = h.Close() }) + + _, err := join(t, h, "ui-trigger-noack", 0, &fakeTrigger{}) + require.NoError(t, err) + + err = h.Ack("ui-trigger-noack", "never-happened") + require.Error(t, err) + assert.True(t, isUserError(err), "%v", err) +} + +// Closing a subscription unregisters every instance and takes it out of the list. +func TestUnsubscribeClosesEverything(t *testing.T) { + h := NewHub() + t.Cleanup(func() { _ = h.Close() }) + + first, second := &fakeTrigger{}, &fakeTrigger{} + _, err := join(t, h, "ui-trigger-close", 0, first) + require.NoError(t, err) + _, err = join(t, h, "ui-trigger-close", 1, second) + require.NoError(t, err) + + require.NoError(t, h.Unsubscribe("ui-trigger-close", nil)) + assert.Len(t, first.unregisters(), 1) + assert.Len(t, second.unregisters(), 1) + assert.Empty(t, h.List()) +} + +// Detaching one instance leaves the rest of the subscription running, which is how +// "stop instance 3 and watch the others" works. +func TestUnsubscribeCanDetachOneInstance(t *testing.T) { + h := NewHub() + t.Cleanup(func() { _ = h.Close() }) + + first, second := &fakeTrigger{}, &fakeTrigger{} + _, err := join(t, h, "ui-trigger-partial", 0, first) + require.NoError(t, err) + _, err = join(t, h, "ui-trigger-partial", 1, second) + require.NoError(t, err) + + require.NoError(t, h.Unsubscribe("ui-trigger-partial", []int{1})) + assert.Empty(t, first.unregisters()) + assert.Len(t, second.unregisters(), 1) + + require.Len(t, h.List(), 1) + assert.Len(t, h.List()[0].Instances, 1) +} + +// A subscription that was never opened is not something the page can watch. +func TestStreamingAnUnknownTriggerIDIsAUserError(t *testing.T) { + h := NewHub() + t.Cleanup(func() { _ = h.Close() }) + + _, err := h.get("ui-trigger-nothing") + require.Error(t, err) + assert.True(t, isUserError(err), "%v", err) +} diff --git a/libs/standalone/protohelpers/ui/metadata.go b/libs/standalone/protohelpers/ui/metadata.go new file mode 100644 index 000000000..dec1da15d --- /dev/null +++ b/libs/standalone/protohelpers/ui/metadata.go @@ -0,0 +1,445 @@ +package ui + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "net/http" + "reflect" + "strconv" + "strings" + "time" + + "github.com/smartcontractkit/chainlink-common/pkg/capabilities" +) + +// HeaderPrefix is what a RequestMetadata field is carried under. One header per +// field, named after the field, so nothing here has to be kept in step with the +// struct by hand: the list below is read off the type. +const HeaderPrefix = "X-CRE-REQUEST-METADATA-" + +// Field is one RequestMetadata field, as the UI needs to render it and as the +// wire carries it. +type Field struct { + // Name is the Go field name, which is what MetadataFromHeaders assigns to. + Name string `json:"name"` + // Header is the HTTP header (and gRPC metadata key) this field travels in. + Header string `json:"header"` + // Kind says which input to draw. It is derived from the Go type, so a field + // gets the same box the proto would give it: text for a string, a number for + // a uint32, a datetime for a timestamp. + Kind string `json:"kind"` + // Repeated fields take one header per entry rather than one header holding + // them all, since http.Header and metadata.MD are both map[string][]string. + Repeated bool `json:"repeated"` + // Default is what an unspecified field is filled in with, shown so the UI can + // pre-populate the box with the value that would be sent anyway. + Default string `json:"default"` +} + +// Kinds a Field can have. Anything unrecognised is text: a box the user can type +// into is a worse fit than a number box, but it is never wrong. +const ( + KindText = "text" + KindNumber = "number" + KindTimestamp = "timestamp" + KindPair = "pair" +) + +// Fields is every RequestMetadata field, in declaration order. +// +// Built by reflection rather than listed, so a field added to RequestMetadata +// upstream shows up here - and in the UI, and on the wire - without this package +// being touched. +func Fields() []Field { + t := reflect.TypeFor[capabilities.RequestMetadata]() + defaults := defaultMetadata() + value := reflect.ValueOf(defaults) + + fields := make([]Field, 0, t.NumField()) + for i := range t.NumField() { + f := t.Field(i) + if !f.IsExported() { + continue + } + kind, repeated := kindOf(f.Type) + fields = append(fields, Field{ + Name: f.Name, + Header: HeaderPrefix + headerSuffix(f.Name), + Kind: kind, + Repeated: repeated, + Default: formatDefault(value.Field(i)), + }) + } + return fields +} + +// kindOf maps a Go type to the input the UI draws for it. +func kindOf(t reflect.Type) (kind string, repeated bool) { + if t == reflect.TypeFor[time.Time]() { + return KindTimestamp, false + } + switch t.Kind() { + case reflect.Slice: + // A slice of two-string tuples (SpendLimit) is a pair per entry. + inner, _ := kindOf(t.Elem()) + if inner == KindText && t.Elem().Kind() == reflect.Struct { + return KindPair, true + } + return inner, true + case reflect.Struct: + // SpendLimit is spend type plus limit, both strings, so one box each. + return KindText, false + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return KindNumber, false + default: + return KindText, false + } +} + +// headerSuffix turns a Go field name into a header suffix: WorkflowID becomes +// WORKFLOW-ID, and WorkflowDonConfigVersion becomes WORKFLOW-DON-CONFIG-VERSION. +// +// Word boundaries are kept because HTTP header names are case-insensitive, so +// camel case alone would not survive canonicalisation - Workflowid and +// WorkflowID are the same header, and neither reads as the field it came from. +func headerSuffix(field string) string { + var words []string + runes := []rune(field) + start := 0 + for i := 1; i <= len(runes); i++ { + if i == len(runes) { + words = append(words, string(runes[start:i])) + break + } + prev, cur := runes[i-1], runes[i] + upper := func(r rune) bool { return r >= 'A' && r <= 'Z' } + switch { + case !upper(prev) && upper(cur): + // camelC -> new word + words = append(words, string(runes[start:i])) + start = i + case upper(prev) && upper(cur) && i+1 < len(runes) && !upper(runes[i+1]): + // end of an acronym run: IDValue -> ID, Value + words = append(words, string(runes[start:i])) + start = i + } + } + for i, w := range words { + words[i] = strings.ToUpper(w) + } + return strings.Join(words, "-") +} + +// Fields the report encoder requires as fixed-length hex. A consensus report +// carries the request metadata on-chain, so these are not free-form strings: the +// encoder decodes them and checks the byte count (see chainlink-common's +// consensus/ocr3/types.Metadata.Encode). +// +// Getting this wrong does not fail the request - it fails the round. The plugin +// cannot encode the metadata, so no report is produced, nothing is transmitted +// back, and the request the user made sits until it expires. What they see is a +// timeout, which says nothing about the value that caused it. +const ( + executionIDBytes = 32 + workflowIDBytes = 32 + workflowNameBytes = 10 + workflowOwnerBytes = 20 +) + +// uiMarker is "ui" in ASCII. A value that has to be hex can still say where it +// came from, so a capability's logs name this page rather than showing an +// anonymous run of zeros. +const uiMarker = "7569" + +// defaultMetadata is what a request gets for anything the caller left out. +// +// Every hex field is a valid one of the right length, because "valid" here means +// what the report encoder accepts rather than what looks reasonable. +// +// The execution identifier is not a constant. A capability may well key work, +// dedupe or cache on the execution it was asked under, so two requests sharing one +// would be two runs of the same execution rather than two executions. Every call +// therefore gets its own, and a fan-out settles on one before sending so its +// instances still agree (see HeadersFromMetadata). +func defaultMetadata() capabilities.RequestMetadata { + return capabilities.RequestMetadata{ + WorkflowID: markedHex(workflowIDBytes), + WorkflowOwner: markedHex(workflowOwnerBytes), + OrgID: "ui-org-id", + WorkflowExecutionID: uniqueHex(executionIDBytes), + WorkflowName: markedHex(workflowNameBytes), + WorkflowDonID: 1, + WorkflowDonConfigVersion: 1, + ReferenceID: "ui-reference-id", + DecodedWorkflowName: "ui-workflow-name", + WorkflowTag: "ui-workflow-tag", + WorkflowRegistryChainSelector: "ui-chain-selector", + WorkflowRegistryAddress: markedHex(workflowOwnerBytes), + EngineVersion: "v1", + ExecutionTimestamp: time.Now().UTC(), + } +} + +// markedHex is a stable hex value of exactly n bytes, opening with the marker so +// it is recognisable in a log. +func markedHex(n int) string { + return pad(uiMarker, n) +} + +// uniqueHex is a per-request hex value of exactly n bytes: the marker, then +// randomness. Random rather than a counter, so two processes - the instances of an +// embed run, say - cannot mint the same one. +func uniqueHex(n int) string { + buf := make([]byte, n) + if _, err := rand.Read(buf); err != nil { + // Randomness is unavailable, which is not worth failing a debug request + // over. A timestamp still separates this from the request before it. + return pad(uiMarker+strconv.FormatInt(time.Now().UnixNano(), 16), n) + } + return pad(uiMarker+hex.EncodeToString(buf), n) +} + +// pad trims or zero-fills a hex string to exactly n bytes, which is the length the +// encoder checks. +func pad(value string, n int) string { + width := n * 2 + if len(value) >= width { + return value[:width] + } + return value + strings.Repeat("0", width-len(value)) +} + +// HeadersFromMetadata renders metadata back into the headers it travels in. +// +// This is what lets a fan-out send one metadata to every instance: it resolves the +// defaults once, then sends the result explicitly rather than letting each +// instance fill in its own - which would give each of them a different execution +// ID for what the user asked for as a single request. +func HeadersFromMetadata(md capabilities.RequestMetadata) http.Header { + header := http.Header{} + value := reflect.ValueOf(md) + t := value.Type() + + for i := range t.NumField() { + f := t.Field(i) + if !f.IsExported() { + continue + } + name := HeaderPrefix + headerSuffix(f.Name) + field := value.Field(i) + + if field.Kind() == reflect.Slice && field.Type() != reflect.TypeFor[time.Time]() { + // One header per entry, which is how they are read back. + for j := range field.Len() { + if entry := formatEntry(field.Index(j)); entry != "" { + header.Add(name, entry) + } + } + continue + } + if rendered := formatDefault(field); rendered != "" { + header.Set(name, rendered) + } + } + return header +} + +// formatEntry renders one element of a repeated field, in the key=value form +// assignEntry reads. +func formatEntry(entry reflect.Value) string { + if entry.Kind() == reflect.String { + return entry.String() + } + if entry.Kind() != reflect.Struct || entry.NumField() != 2 { + return "" + } + first, second := entry.Field(0), entry.Field(1) + if first.Kind() != reflect.String || second.Kind() != reflect.String { + return "" + } + return first.String() + "=" + second.String() +} + +// formatDefault renders a default the way the UI would show it, and the way +// MetadataFromHeaders parses it back. +func formatDefault(v reflect.Value) string { + switch { + case v.Type() == reflect.TypeFor[time.Time](): + t := v.Interface().(time.Time) + if t.IsZero() { + return "" + } + return t.UTC().Format(time.RFC3339) + case v.Kind() == reflect.Slice: + return "" + case v.CanUint(): + return strconv.FormatUint(v.Uint(), 10) + case v.CanInt(): + return strconv.FormatInt(v.Int(), 10) + default: + return fmt.Sprint(v.Interface()) + } +} + +// MetadataFromHeaders builds the metadata a capability is called with from the +// headers a request carried. +// +// get is passed the header name and returns every value under it, so this works +// against an http.Header and against gRPC metadata.MD alike - both are +// map[string][]string, and a repeated field is repeated headers rather than one +// header holding a list. +// +// Anything absent or blank is filled in from defaultMetadata, so a caller that +// specifies nothing still sends a usable request. An unparseable number or +// timestamp is an error rather than a silent default: the caller asked for +// something specific and got neither it nor a warning otherwise. +func MetadataFromHeaders(get func(name string) []string) (capabilities.RequestMetadata, error) { + metadata := defaultMetadata() + out := reflect.ValueOf(&metadata).Elem() + t := out.Type() + + for i := range t.NumField() { + f := t.Field(i) + if !f.IsExported() { + continue + } + values := nonBlank(get(HeaderPrefix + headerSuffix(f.Name))) + if len(values) == 0 { + continue + } + if err := assign(out.Field(i), values); err != nil { + return capabilities.RequestMetadata{}, userErrorf("%s: %s", HeaderPrefix+headerSuffix(f.Name), err) + } + } + + // The execution timestamp is the one default that cannot be a constant: a + // fixed one would put every request at the same instant. + if metadata.ExecutionTimestamp.IsZero() { + metadata.ExecutionTimestamp = time.Now().UTC() + } + + if err := validateHexFields(metadata); err != nil { + return capabilities.RequestMetadata{}, err + } + return metadata, nil +} + +// validateHexFields rejects a value the report encoder would refuse. +// +// Checked here so it is reported as the field it is. Left to the capability, the +// same mistake is not an error at all: the consensus plugin cannot encode the +// metadata, so it produces no report, nothing is transmitted back, and the +// request the user made sits until it expires. What they are told is "timeout +// exceeded", which names neither the field nor the length - and takes twenty +// seconds to say it. +func validateHexFields(md capabilities.RequestMetadata) error { + for _, f := range []struct { + name string + value string + bytes int + }{ + {"WorkflowExecutionID", md.WorkflowExecutionID, executionIDBytes}, + {"WorkflowID", md.WorkflowID, workflowIDBytes}, + {"WorkflowName", md.WorkflowName, workflowNameBytes}, + {"WorkflowOwner", md.WorkflowOwner, workflowOwnerBytes}, + } { + decoded, err := hex.DecodeString(strings.TrimPrefix(f.value, "0x")) + if err != nil { + return userErrorf("%s%s: %s must be hex, because a consensus report carries it on-chain: %s", + HeaderPrefix, headerSuffix(f.name), f.name, err) + } + // WorkflowName is padded to length by the encoder, so a short one is fine. + if len(decoded) > f.bytes || (len(decoded) < f.bytes && f.name != "WorkflowName") { + return userErrorf("%s%s: %s must be %d hex bytes (%d characters), got %d", + HeaderPrefix, headerSuffix(f.name), f.name, f.bytes, f.bytes*2, len(decoded)) + } + } + return nil +} + +func nonBlank(values []string) []string { + out := make([]string, 0, len(values)) + for _, v := range values { + if strings.TrimSpace(v) != "" { + out = append(out, v) + } + } + return out +} + +// assign writes header values into one field, by its Go type. +func assign(field reflect.Value, values []string) error { + switch { + case field.Type() == reflect.TypeFor[time.Time](): + parsed, err := time.Parse(time.RFC3339, values[0]) + if err != nil { + return fmt.Errorf("expected an RFC3339 timestamp: %w", err) + } + field.Set(reflect.ValueOf(parsed)) + return nil + + case field.Kind() == reflect.Slice: + slice := reflect.MakeSlice(field.Type(), 0, len(values)) + for _, v := range values { + entry := reflect.New(field.Type().Elem()).Elem() + if err := assignEntry(entry, v); err != nil { + return err + } + slice = reflect.Append(slice, entry) + } + field.Set(slice) + return nil + + case field.CanUint(): + n, err := strconv.ParseUint(values[0], 10, 64) + if err != nil { + return fmt.Errorf("expected a number: %w", err) + } + field.SetUint(n) + return nil + + case field.CanInt(): + n, err := strconv.ParseInt(values[0], 10, 64) + if err != nil { + return fmt.Errorf("expected a number: %w", err) + } + field.SetInt(n) + return nil + + case field.Kind() == reflect.String: + field.SetString(values[0]) + return nil + + default: + return fmt.Errorf("unsupported field type %s", field.Type()) + } +} + +// assignEntry fills one element of a repeated field from a single header value. +// +// A two-string struct (SpendLimit) is "type=limit", the same key=value form the +// standalone config uses for its own pair-valued settings. +func assignEntry(entry reflect.Value, value string) error { + if entry.Kind() == reflect.String { + entry.SetString(value) + return nil + } + if entry.Kind() != reflect.Struct || entry.NumField() != 2 { + return fmt.Errorf("unsupported repeated element type %s", entry.Type()) + } + + key, limit, found := strings.Cut(value, "=") + if !found || key == "" { + return fmt.Errorf("invalid entry %q: expected key=value", value) + } + for i, v := range []string{key, limit} { + f := entry.Field(i) + if f.Kind() != reflect.String { + return fmt.Errorf("unsupported repeated element type %s", entry.Type()) + } + f.SetString(v) + } + return nil +} diff --git a/libs/standalone/protohelpers/ui/metadata_test.go b/libs/standalone/protohelpers/ui/metadata_test.go new file mode 100644 index 000000000..84453664a --- /dev/null +++ b/libs/standalone/protohelpers/ui/metadata_test.go @@ -0,0 +1,274 @@ +package ui + +import ( + "encoding/hex" + "net/http" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-common/pkg/capabilities" + ocr3types "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3/types" +) + +func TestHeaderSuffix(t *testing.T) { + for field, want := range map[string]string{ + "WorkflowID": "WORKFLOW-ID", + "OrgID": "ORG-ID", + "WorkflowDonConfigVersion": "WORKFLOW-DON-CONFIG-VERSION", + "SpendLimits": "SPEND-LIMITS", + "ExecutionTimestamp": "EXECUTION-TIMESTAMP", + "WorkflowRegistryChainSelector": "WORKFLOW-REGISTRY-CHAIN-SELECTOR", + } { + assert.Equal(t, want, headerSuffix(field), field) + } +} + +// Every field of RequestMetadata gets a header, so a field added upstream is +// carried without this package being touched. +func TestFieldsCoverEveryMetadataField(t *testing.T) { + fields := Fields() + require.NotEmpty(t, fields) + + byName := map[string]Field{} + for _, f := range fields { + byName[f.Name] = f + } + + for _, name := range []string{ + "WorkflowID", "WorkflowOwner", "OrgID", "WorkflowExecutionID", "WorkflowName", + "WorkflowDonID", "WorkflowDonConfigVersion", "ReferenceID", "DecodedWorkflowName", + "SpendLimits", "WorkflowTag", "WorkflowRegistryChainSelector", + "WorkflowRegistryAddress", "EngineVersion", "ExecutionTimestamp", + } { + f, ok := byName[name] + require.True(t, ok, "%s has no header", name) + assert.True(t, len(f.Header) > len(HeaderPrefix), "%s: %q", name, f.Header) + } + + assert.Equal(t, KindNumber, byName["WorkflowDonID"].Kind) + assert.Equal(t, KindText, byName["WorkflowID"].Kind) + assert.Equal(t, KindTimestamp, byName["ExecutionTimestamp"].Kind) + assert.True(t, byName["SpendLimits"].Repeated, "spend limits take one header per entry") +} + +// Nothing specified means a request a capability will accept rather than one it +// rejects for a missing field. +func TestMetadataDefaultsWhenNothingIsSpecified(t *testing.T) { + before := time.Now().UTC() + md, err := MetadataFromHeaders(func(string) []string { return nil }) + require.NoError(t, err) + + assert.Equal(t, markedHex(workflowIDBytes), md.WorkflowID) + assert.Equal(t, markedHex(workflowOwnerBytes), md.WorkflowOwner) + assert.NotContains(t, md.WorkflowID, "test", "the defaults are the UI's, not a test's") + assert.EqualValues(t, 1, md.WorkflowDonID) + assert.EqualValues(t, 1, md.WorkflowDonConfigVersion) + assert.NotEmpty(t, md.WorkflowExecutionID) + + // The timestamp is the one default that cannot be a constant. + assert.False(t, md.ExecutionTimestamp.Before(before), "%s", md.ExecutionTimestamp) +} + +func TestMetadataFromHeaders(t *testing.T) { + header := http.Header{} + header.Set(HeaderPrefix+"WORKFLOW-ID", strings.Repeat("cd", workflowIDBytes)) + header.Set(HeaderPrefix+"WORKFLOW-DON-ID", "7") + header.Set(HeaderPrefix+"EXECUTION-TIMESTAMP", "2026-08-19T10:11:12Z") + // Repeated is repeated headers, since a header can hold a list. + header.Add(HeaderPrefix+"SPEND-LIMITS", "CONSENSUS=100000") + header.Add(HeaderPrefix+"SPEND-LIMITS", "COMPUTE=5") + + md, err := MetadataFromHeaders(header.Values) + require.NoError(t, err) + + assert.Equal(t, strings.Repeat("cd", workflowIDBytes), md.WorkflowID) + assert.EqualValues(t, 7, md.WorkflowDonID) + assert.Equal(t, "2026-08-19T10:11:12Z", md.ExecutionTimestamp.Format(time.RFC3339)) + assert.Equal(t, []capabilities.SpendLimit{ + {SpendType: "CONSENSUS", Limit: "100000"}, + {SpendType: "COMPUTE", Limit: "5"}, + }, md.SpendLimits) + + // Anything not sent still falls back. + assert.Equal(t, markedHex(workflowOwnerBytes), md.WorkflowOwner) + assert.EqualValues(t, 1, md.WorkflowDonConfigVersion) +} + +// A blank header is the same as an absent one: the UI leaves empty boxes off the +// wire, and an empty value would otherwise override the default with nothing. +func TestBlankHeadersFallBackToDefaults(t *testing.T) { + header := http.Header{} + header.Set(HeaderPrefix+"WORKFLOW-ID", " ") + + md, err := MetadataFromHeaders(header.Values) + require.NoError(t, err) + assert.Equal(t, markedHex(workflowIDBytes), md.WorkflowID) +} + +// A value the caller asked for that cannot be parsed is an error, not a silent +// default: they would otherwise get neither their value nor a warning. +func TestUnparseableValuesAreErrors(t *testing.T) { + for name, value := range map[string]string{ + "WORKFLOW-DON-ID": "not-a-number", + "EXECUTION-TIMESTAMP": "yesterday", + "SPEND-LIMITS": "no-equals-sign", + } { + header := http.Header{} + header.Set(HeaderPrefix+name, value) + + _, err := MetadataFromHeaders(header.Values) + require.Error(t, err, name) + assert.Contains(t, err.Error(), name) + } +} + +func TestHeaderNamesArePrefixed(t *testing.T) { + names := HeaderNames() + require.Len(t, names, len(Fields())) + for _, n := range names { + assert.Contains(t, n, HeaderPrefix) + } +} + +// Two requests are two executions, so the identifier a capability might key work +// on cannot be shared between them. +func TestExecutionIDIsUniquePerRequest(t *testing.T) { + first, err := MetadataFromHeaders(func(string) []string { return nil }) + require.NoError(t, err) + second, err := MetadataFromHeaders(func(string) []string { return nil }) + require.NoError(t, err) + + assert.NotEqual(t, first.WorkflowExecutionID, second.WorkflowExecutionID) + assert.True(t, strings.HasPrefix(first.WorkflowExecutionID, uiMarker)) + + // The workflow itself is the same workflow, so that one is stable. + assert.Equal(t, first.WorkflowID, second.WorkflowID) +} + +// A caller who names an execution gets the one they named. +func TestExecutionIDCanBeSpecified(t *testing.T) { + header := http.Header{} + mine := strings.Repeat("ef", executionIDBytes) + header.Set(HeaderPrefix+"WORKFLOW-EXECUTION-ID", mine) + + md, err := MetadataFromHeaders(header.Values) + require.NoError(t, err) + assert.Equal(t, mine, md.WorkflowExecutionID) +} + +// Rendering metadata back to headers and reading it again returns what went in. +// This is what a fan-out relies on to send one metadata to several instances. +func TestHeadersRoundTrip(t *testing.T) { + original, err := MetadataFromHeaders(func(string) []string { return nil }) + require.NoError(t, err) + original.SpendLimits = []capabilities.SpendLimit{ + {SpendType: "CONSENSUS", Limit: "100000"}, + {SpendType: "COMPUTE", Limit: "5"}, + } + + header := HeadersFromMetadata(original) + roundTripped, err := MetadataFromHeaders(header.Values) + require.NoError(t, err) + + assert.Equal(t, original.WorkflowExecutionID, roundTripped.WorkflowExecutionID) + assert.Equal(t, original.WorkflowID, roundTripped.WorkflowID) + assert.Equal(t, original.WorkflowDonID, roundTripped.WorkflowDonID) + assert.Equal(t, original.SpendLimits, roundTripped.SpendLimits) + assert.Equal(t, + original.ExecutionTimestamp.Format(time.RFC3339), + roundTripped.ExecutionTimestamp.Format(time.RFC3339)) +} + +// A value that will not parse is the caller's to fix, so it is a 400 rather than +// the page reporting itself as broken. +func TestUnparseableValuesAreUserErrors(t *testing.T) { + header := http.Header{} + header.Set(HeaderPrefix+"WORKFLOW-DON-ID", "not-a-number") + + _, err := MetadataFromHeaders(header.Values) + require.Error(t, err) + assert.True(t, isUserError(err), "%v", err) + assert.Equal(t, http.StatusBadRequest, httpStatus(err)) +} + +// The defaults have to satisfy the report encoder, not merely look plausible. +// +// This is the check that was missing. Values the encoder rejects do not fail the +// request: they fail the consensus round, so no report is produced, nothing is +// transmitted back, and the request expires. The user sees a timeout that says +// nothing about the field that caused it. +func TestDefaultsSatisfyTheReportEncoder(t *testing.T) { + md, err := MetadataFromHeaders(func(string) []string { return nil }) + require.NoError(t, err) + + encoded, err := ocr3types.Metadata{ + Version: 1, + ExecutionID: md.WorkflowExecutionID, + Timestamp: uint32(md.ExecutionTimestamp.Unix()), + DONID: md.WorkflowDonID, + DONConfigVersion: md.WorkflowDonConfigVersion, + WorkflowID: md.WorkflowID, + WorkflowName: md.WorkflowName, + WorkflowOwner: md.WorkflowOwner, + ReportID: "0000", + }.Encode() + require.NoError(t, err, "the defaults must encode: a value the encoder rejects surfaces as a timeout") + assert.NotEmpty(t, encoded) +} + +// The hex fields are the exact byte lengths the encoder checks. +func TestHexDefaultsHaveTheRequiredLengths(t *testing.T) { + md, err := MetadataFromHeaders(func(string) []string { return nil }) + require.NoError(t, err) + + for name, tc := range map[string]struct { + value string + bytes int + }{ + "WorkflowExecutionID": {md.WorkflowExecutionID, executionIDBytes}, + "WorkflowID": {md.WorkflowID, workflowIDBytes}, + "WorkflowName": {md.WorkflowName, workflowNameBytes}, + "WorkflowOwner": {md.WorkflowOwner, workflowOwnerBytes}, + } { + t.Run(name, func(t *testing.T) { + decoded, err := hex.DecodeString(tc.value) + require.NoError(t, err, "%s must be hex, got %q", name, tc.value) + assert.Len(t, decoded, tc.bytes) + // Still says where it came from, despite having to be hex. + assert.True(t, strings.HasPrefix(tc.value, uiMarker), "%s: %q", name, tc.value) + }) + } +} + +// A value the encoder would refuse is reported as the field it is, immediately, +// rather than becoming a consensus timeout twenty seconds later. +func TestHexFieldsAreValidatedUpFront(t *testing.T) { + for name, value := range map[string]string{ + "WORKFLOW-ID": "not-hex", + "WORKFLOW-EXECUTION-ID": "also-not-hex", + "WORKFLOW-OWNER": "0xzz", + } { + t.Run(name, func(t *testing.T) { + header := http.Header{} + header.Set(HeaderPrefix+name, value) + + _, err := MetadataFromHeaders(header.Values) + require.Error(t, err) + assert.True(t, isUserError(err), "%v", err) + assert.Contains(t, err.Error(), name, "the message should name the field") + }) + } + + t.Run("the right length is required", func(t *testing.T) { + header := http.Header{} + header.Set(HeaderPrefix+"WORKFLOW-ID", "abcd") // hex, but 2 bytes not 32 + + _, err := MetadataFromHeaders(header.Values) + require.Error(t, err) + assert.Contains(t, err.Error(), "32 hex bytes") + }) +} diff --git a/libs/standalone/protohelpers/ui/mount.go b/libs/standalone/protohelpers/ui/mount.go new file mode 100644 index 000000000..a93a6dda8 --- /dev/null +++ b/libs/standalone/protohelpers/ui/mount.go @@ -0,0 +1,488 @@ +package ui + +import ( + "crypto/rand" + "encoding/base64" + "encoding/json" + "fmt" + "html/template" + "io" + "net/http" + "path" + "sort" + "strings" + "sync" + + "github.com/fullstorydev/grpcui/standalone" +) + +// grpcui keeps these unexported, so the names are repeated here: the fan-out +// speaks the same CSRF scheme as the page it calls into. +const ( + csrfCookieName = "_grpcui_csrf_token" + csrfHeaderName = "x-grpcui-csrf-token" + + // The fan-out page gets its own names. grpcui scopes its cookie to the page's + // own path, so a same-named cookie higher up would be sent alongside it and + // could shadow the page's token. + fanoutCookieName = "_cre_debug_csrf_token" + fanoutHeaderName = "x-cre-debug-csrf-token" +) + +// DefaultPrefix is where the debug pages are mounted. +const DefaultPrefix = "/debug/capabilities" + +// Options is what mounting one instance's debug page needs. +type Options struct { + // Mux is what the pages are served on: the instance's own HTTP server, so a + // browser on any instance's port can drive the whole process. + Mux *http.ServeMux + // Prefix roots both pages. Empty means DefaultPrefix. + Prefix string + // Server is this instance's capabilities. + Server *Server + // Fleet is every instance's page, so the fan-out can reach a sibling. + Fleet *Fleet + // Hub holds the subscriptions. Shared with every other instance, so a trigger + // registered across several of them is one subscription with a column each. + Hub *Hub + // Index is this instance's number, which names it on the fan-out page and on + // every event it delivers. + Index int + // Title is what the per-instance page calls itself. + Title string +} + +// Mount serves this instance's debug page, and adds it to the fleet so the +// fan-out page can reach it. +// +// prefix roots both pages: prefix+"/ui/" is the form for this instance's own +// capabilities, and prefix+"/request" is the fan-out over every instance. Both are +// mounted on every instance, so whichever port a browser lands on can drive the +// whole process. +func Mount(o Options) error { + if o.Mux == nil { + return fmt.Errorf("a mux is required") + } + if o.Server == nil { + return fmt.Errorf("a server is required") + } + if o.Fleet == nil { + return fmt.Errorf("a fleet is required") + } + if o.Hub == nil { + return fmt.Errorf("a subscription hub is required") + } + + prefix := o.Prefix + if prefix == "" { + prefix = DefaultPrefix + } + prefix = "/" + strings.Trim(prefix, "/") + + mux, s := o.Mux, o.Server + label := fmt.Sprintf("instance %d", o.Index+1) + + // Which instance this is, and where its subscriptions go. Set here rather than + // passed to New because this is where an instance's identity is known: New is + // given capabilities, and a capability does not know which instance is hosting + // it. + s.hub = o.Hub + s.index = o.Index + s.label = label + s.prefix = prefix + + served, err := customJS(s) + if err != nil { + return err + } + + page := standalone.Handler( + s, + o.Title, + s.Methods(), + s.Files(), + standalone.AddCSSFile(cssFileName(), func() (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader(pageCSS)), nil + }), + standalone.AddJSFile(jsFileName(served), func() (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader(served)), nil + }), + // The metadata a capability is called with comes from the browser, and so + // does the trigger ID a subscription is identified by, so every header + // either travels in has to reach Invoke. + standalone.PreserveHeaders(PreservedHeaders()), + ) + + uiPath := prefix + "/ui" + mux.Handle(uiPath+"/", http.StripPrefix(uiPath, page)) + + o.Fleet.Add(&Instance{ + Index: o.Index, + Label: label, + handler: page, + }) + + f := &fanout{fleet: o.Fleet, hub: o.Hub, prefix: prefix, uiPath: uiPath, server: s} + mux.HandleFunc(prefix+"/request", f.page) + mux.HandleFunc(prefix+"/request/", f.page) + mux.HandleFunc(prefix+"/request/fanout", f.invoke) + mux.HandleFunc(prefix+"/request/s/", f.asset) + + // The subscriptions: what is running, what they have delivered, and the two + // things a reader does about it. + mux.HandleFunc(prefix+"/request/subscriptions", f.subscriptions) + mux.HandleFunc(prefix+"/request/subscriptions/stream", f.stream) + mux.HandleFunc(prefix+"/request/subscriptions/ack", f.ack) + mux.HandleFunc(prefix+"/request/subscriptions/close", f.unsubscribe) + mux.HandleFunc(prefix+"/request/trigger-id", f.triggerID) + + // A bare prefix is otherwise a 404, which reads as the page not being there. + mux.HandleFunc(prefix, func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, uiPath+"/", http.StatusFound) + }) + return nil +} + +// fanout serves the page that sends one or more requests across every instance. +type fanout struct { + fleet *Fleet + hub *Hub + prefix string + uiPath string + server *Server + + once sync.Once + template *template.Template + tmplErr error +} + +// pageConfig is what request.js needs to build the page. +type pageConfig struct { + Instances []*Instance `json:"instances"` + Services map[string][]string `json:"services"` + UIPath string `json:"uiPath"` + Prefix string `json:"prefix"` + // Metadata is every RequestMetadata field, which is what the page's Advanced + // section is built from rather than a hand-written list of inputs. + Metadata []Field `json:"metadata"` + + // Subscriptions are the services whose methods register a trigger rather than + // calling it. Invoking one opens a subscription instead of returning a + // response, so the page has to know which it is looking at. + Subscriptions []string `json:"subscriptions"` + // TriggerIDHeader is what the trigger ID travels in, so the page does not + // repeat the name. + TriggerIDHeader string `json:"triggerIdHeader"` + // Special are the messages shown as the number they stand for rather than as + // the fields they are made of, and where each method's response holds them. + // See special.go. + Special SpecialConfig `json:"special"` +} + +func (f *fanout) config() pageConfig { + services := map[string][]string{} + for key := range f.server.calls { + service, method, found := strings.Cut(key, "/") + if !found { + continue + } + services[service] = append(services[service], method) + } + for _, methods := range services { + sort.Strings(methods) + } + + return pageConfig{ + Instances: f.fleet.List(), + Services: services, + UIPath: f.uiPath, + Prefix: f.prefix, + Metadata: Fields(), + Subscriptions: f.server.subscriptionServices(), + TriggerIDHeader: TriggerIDHeader, + Special: f.server.specialConfig(), + } +} + +func (f *fanout) page(w http.ResponseWriter, r *http.Request) { + f.once.Do(func() { + f.template, f.tmplErr = template.New("request.html").Parse(requestHTML) + }) + if f.tmplErr != nil { + writeError(w, systemErrorf("the debug page template is invalid: %w", f.tmplErr)) + return + } + + ensureCSRFCookie(w, r) + + encoded, err := json.Marshal(f.config()) + if err != nil { + writeError(w, systemErrorf("failed to encode the debug page config: %w", err)) + return + } + + w.Header().Set("Content-Type", "text/html; charset=utf-8") + // Must revalidate, or a rebuild would not reach an open browser. + w.Header().Set("Cache-Control", "no-cache, must-revalidate") + + data := struct { + CSSFile string + JSFile string + SubscriptionsFile string + UIPath string + Prefix string + Config template.JS + }{ + CSSFile: cssFileName(), + JSFile: requestJSFileName(), + SubscriptionsFile: subscriptionsJSFileName(), + UIPath: f.uiPath, + Prefix: f.prefix, + Config: template.JS(encoded), + } + if err := f.template.Execute(w, data); err != nil { + return + } +} + +// asset serves the fan-out page's own script, and the shared stylesheet, under +// their content-hashed names. +func (f *fanout) asset(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "private, max-age=3600") + switch path.Base(r.URL.Path) { + case requestJSFileName(): + w.Header().Set("Content-Type", "text/javascript; charset=utf-8") + // The shared arithmetic first, so the page's own script can rely on it. + _, _ = io.WriteString(w, valuesJS+"\n"+requestJS) + case subscriptionsJSFileName(): + w.Header().Set("Content-Type", "text/javascript; charset=utf-8") + _, _ = io.WriteString(w, subscriptionsJS) + case cssFileName(): + w.Header().Set("Content-Type", "text/css; charset=utf-8") + _, _ = io.WriteString(w, pageCSS) + default: + http.NotFound(w, r) + } +} + +// requestGroup is one request body plus the instances it is addressed to. +type requestGroup struct { + Instances []int `json:"instances"` + Body json.RawMessage `json:"body"` +} + +type fanoutRequest struct { + // Method is dot-separated, the way grpcui's invoke route expects it. + Method string `json:"method"` + Groups []requestGroup `json:"groups"` + // Metadata is the request metadata every group is sent with. The page fills it + // in before sending, so each instance is called with the same metadata and a + // difference between their answers is the capability's rather than the call's. + Metadata map[string][]string `json:"metadata"` +} + +// instanceResult is one instance's answer. Status is "ok" when the instance +// answered, "error" when the call failed, and "na" when it was not addressed. +type instanceResult struct { + Instance int `json:"instance"` + Label string `json:"label"` + Status string `json:"status"` + Group int `json:"group"` + // ResponseID is the hash of what this instance answered, and ResponseIndex is + // which of the fan-out's distinct responses that is. + // + // The response itself is on the fan-out rather than here, for the same reason a + // trigger event's payload is on its row: instances answering identically is the + // normal case, and holding it per instance would repeat the same JSON once per + // instance to say they matched. + ResponseID string `json:"responseId,omitempty"` + ResponseIndex int `json:"responseIndex"` + Error string `json:"error,omitempty"` +} + +type fanoutResponse struct { + Method string `json:"method"` + Results []instanceResult `json:"results"` + // TriggerID is the subscription every group was registered under, for a + // fan-out that subscribed rather than called. Reported back because the page + // needs it to open the stream, and because a caller that named none still has + // to be told which one it got. + TriggerID string `json:"triggerId,omitempty"` + + // The distinct responses, and whether the instances disagreed. Same shape as a + // trigger event's row, because it is the same question: what did each instance + // say, and did they all say it. + payloadSet +} + +func (f *fanout) invoke(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "POST only", http.StatusMethodNotAllowed) + return + } + // The same CSRF shape grpcui uses, so this is no easier to drive from a + // hostile page than the pages behind it. + cookie, err := r.Cookie(fanoutCookieName) + if err != nil || cookie.Value == "" || cookie.Value != r.Header.Get(fanoutHeaderName) { + http.Error(w, "incorrect CSRF token", http.StatusUnauthorized) + return + } + + var req fanoutRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, userErrorf("bad request body: %s", err)) + return + } + if req.Method == "" { + writeError(w, userErrorf("method is required")) + return + } + + // Resolved here, once, rather than letting each instance fill in its own + // defaults: the unspecified fields include the execution ID, so instances left + // to their own devices would each invent a different one and what the user + // asked for as a single request would arrive as several. Doing it here also + // means a value that will not parse is one 400 rather than the same complaint + // from every instance. + get := func(name string) []string { return req.Metadata[name] } + + metadata, err := MetadataFromHeaders(get) + if err != nil { + writeError(w, err) + return + } + header := HeadersFromMetadata(metadata) + + // Settled here for the same reason, and it matters more: the trigger ID is + // what identifies a subscription, so instances left to mint their own would + // each start a subscription of their own and the one table the user asked for + // would be four. + triggerID := TriggerIDFromHeaders(get) + header.Set(TriggerIDHeader, triggerID) + + response := f.run(req, header) + response.TriggerID = triggerID + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(response); err != nil { + return + } +} + +// run sends each group to the instances it names, and returns one row per +// instance so an unaddressed one still reports N/A. +// +// Every instance is called at once, not one after another. An OCR capability +// cannot answer one instance until enough of the others have joined the same +// round, so asking them in turn would leave the first waiting for a quorum that +// has not been invited yet - the call would time out rather than return. Even for +// a capability that could answer alone, "all of them at once" is what a fan-out +// is for. +func (f *fanout) run(req fanoutRequest, header http.Header) fanoutResponse { + all := f.fleet.List() + byIndex := map[int]*Instance{} + for _, in := range all { + byIndex[in.Index] = in + } + + // The work is collected before any of it runs, so the whole fan-out starts + // together rather than a group at a time. + type job struct { + instance *Instance + group int + body []byte + } + var jobs []job + claimed := map[int]bool{} + for gi, group := range req.Groups { + for _, idx := range group.Instances { + in, ok := byIndex[idx] + if !ok || claimed[idx] { + // Unknown instance, or one an earlier group already claimed: the + // page keeps the groups disjoint, this is the backstop. + continue + } + claimed[idx] = true + jobs = append(jobs, job{instance: in, group: gi + 1, body: group.Body}) + } + } + + type answer struct { + result instanceResult + response json.RawMessage + } + + answers := make(map[int]answer, len(jobs)) + var mu sync.Mutex + var wg sync.WaitGroup + for _, j := range jobs { + wg.Add(1) + go func(j job) { + defer wg.Done() + + row := instanceResult{ + Instance: j.instance.Index, + Label: j.instance.Label, + Group: j.group, + Status: "ok", + } + response, err := j.instance.invoke(req.Method, j.body, header) + if err != nil { + row.Status = "error" + row.Error = err.Error() + response = nil + } + + mu.Lock() + defer mu.Unlock() + answers[j.instance.Index] = answer{result: row, response: response} + }(j) + } + wg.Wait() + + // Collected in instance order rather than as they arrived, so the responses are + // numbered the same way twice in a row for the same fan-out. Concurrency makes + // arrival order arbitrary, and a debug page that renumbers its columns between + // two identical runs is a page that looks like it found something. + out := fanoutResponse{Method: req.Method, Results: make([]instanceResult, 0, len(all))} + for _, in := range all { + got, ok := answers[in.Index] + if !ok { + out.Results = append(out.Results, instanceResult{ + Instance: in.Index, + Label: in.Label, + Status: "na", + ResponseIndex: -1, + }) + continue + } + + // The hash is of the bytes the instance's page produced, which is what the + // form generator rendered and what is about to be shown. + row := got.result + if len(got.response) > 0 { + row.ResponseID = shortHash(got.response) + } + row.ResponseIndex = out.add(row.ResponseID, got.response) + out.Results = append(out.Results, row) + } + return out +} + +// ensureCSRFCookie mirrors what grpcui does for its own pages, so the fan-out page +// has a token to send back. +func ensureCSRFCookie(w http.ResponseWriter, r *http.Request) { + if _, err := r.Cookie(fanoutCookieName); err == nil { + return + } + buf := make([]byte, 32) + if _, err := rand.Read(buf); err != nil { + return + } + http.SetCookie(w, &http.Cookie{ + Name: fanoutCookieName, + Value: base64.RawURLEncoding.EncodeToString(buf), + Path: "/", + }) +} diff --git a/libs/standalone/protohelpers/ui/resources/form.js b/libs/standalone/protohelpers/ui/resources/form.js new file mode 100644 index 000000000..dd2a25c97 --- /dev/null +++ b/libs/standalone/protohelpers/ui/resources/form.js @@ -0,0 +1,817 @@ +// Per-instance capability form. +// +// grpcui renders the form; this adds what a capability call needs on top of it: +// every optional field marked present, the request metadata a host would have +// carried made editable, and - when embedded in the fan-out page - hooks for the +// parent to read and write a request body. +// +// Everything type-related comes from window.__CRE_DEBUG__, which the server builds +// from the RequestMetadata type and the capability descriptors. Nothing here +// infers a type from the shape of the data. + +document.addEventListener("DOMContentLoaded", function () { + var CONFIG = window.__CRE_DEBUG__ || { + metadata: [], headerPrefix: "", subscriptions: [], triggerIdHeader: "", prefix: "", + special: { bigInt: "", decimal: "", methods: {} } + }; + + // A method on one of these services registers a trigger rather than calling + // something, so it takes a trigger ID and its events turn up on the fan-out + // page rather than in the Response tab. + function isSubscribing() { + return (CONFIG.subscriptions || []).indexOf($("#grpc-service").val()) !== -1; + } + + // ---- request metadata ---------------------------------------------------- + // + // The metadata travels as one header per field, which grpcui forwards because + // the server passed those names to PreserveHeaders. Each field is mirrored + // into a row of grpcui's own (hidden) metadata table, so it rides along with + // the request the same way a hand-typed header would. + + function ensureAdvancedSection() { + if ($("#cre-advanced").length) { + return; + } + var $metadata = $("#grpc-request-metadata"); + var $invoke = $("#grpc-request-tab > button.grpc-invoke").first(); + if (!$metadata.length || !$invoke.length) { + return; + } + + // The "Request Metadata" h3 has no id, so tag it for the CSS to hide. + $metadata.prev("h3").addClass("cre-hidden"); + + var $details = $("
", { id: "cre-advanced" }); + $details.append($("", { text: "Advanced" })); + $details.append($("
", { + "class": "cre-metadata-note", + text: "Request metadata. Anything left blank is filled in with a valid value by the server." + })); + + var $grid = $("
", { "class": "cre-metadata" }); + CONFIG.metadata.forEach(function (field) { + var $row = $("", { "class": "cre-metadata-field" }); + $row.append($("