feat(metrics): count provider calls and record latency by endpoint and outcome

src/__tests__/linq/providerCalls.test.ts

@@ -0,0 +1,197 @@
+/**
+ * Tests for provider_calls_total and Linq call outcome metrics.
+ *
+ * Requirements:
+ * - Counter of provider calls by endpoint path and outcome, incremented in src/linq/http.ts
+ * - Every call through linqCall is counted, with no per-rail opt-in
+ * - Labels are bounded: endpoint path and outcome only
+ * - Outcome distinguishes: success, 4xx, 5xx, 429, timeout
+ * - Latency is observed for failures as well as successes
+ * - Timeout is recorded as timeout, not generic failure
+ * - Tested against fake provider (never real Linq)
+ */
+import { linqCall } from "../../linq/http";
+import { renderMetrics, _resetMetrics } from "../../metrics";
+import { installFakeLinq, type FakeLinq } from "../fakes/linqProvider";
+
+process.env.LINQ_API_KEY = "biz_live_test";
+
+let linq: FakeLinq;
+
+beforeEach(() => {
+  _resetMetrics();
+  linq = installFakeLinq();
+  jest.spyOn(console, "warn").mockImplementation(() => {});
+});
+
+afterEach(() => {
+  linq.restore();
+  jest.restoreAllMocks();
+});
+
+function series(
+  text: string,
+  metric: string,
+  labels: Record<string, string>,
+): number | undefined {
+  const line = text.split("\n").find(
+    (candidate) =>
+      candidate.startsWith(`${metric}{`) &&
+      Object.entries(labels).every(([key, value]) => candidate.includes(`${key}="${value}"`)),
+  );
+  return line === undefined ? undefined : Number(line.slice(line.lastIndexOf(" ") + 1));
+}
+
+function sampleHistogram(
+  text: string,
+  suffix: "count" | "sum",
+  labels: Record<string, string>,
+): number | undefined {
+  const line = text.split("\n").find(
+    (candidate) =>
+      candidate.startsWith(`provider_request_duration_seconds_${suffix}{`) &&
+      Object.entries(labels).every(([key, value]) => candidate.includes(`${key}="${value}"`)),
+  );
+  return line === undefined ? undefined : Number(line.slice(line.lastIndexOf(" ") + 1));
+}
+
+describe("provider_calls_total", () => {
+  it("counts a successful call by endpoint path and outcome", async () => {
+    await linqCall("/b2b/customers", {
+      method: "POST",
+      operation: "onramp_customer",
+      body: { customerRef: "c_test_1" },
+    });
+
+    const text = await renderMetrics();
+    expect(series(text, "provider_calls_total", { endpoint: "/b2b/customers", outcome: "success" })).toBe(1);
+    expect(sampleHistogram(text, "count", { provider: "linq", operation: "onramp_customer", outcome: "success" })).toBe(1);
+  });
+
+  it("normalizes paths by stripping query parameters to preserve bounded cardinality", async () => {
+    // Calling an endpoint with dynamic query parameters
+    linq.failNext("/b2b/status", {
+      kind: "refuse",
+      status: 404,
+      body: { message: "Order not found" },
+    });
+
+    await expect(
+      linqCall("/b2b/status?id=ord_xyz_123&foo=bar", {
+        operation: "offramp_status",
+      }),
+    ).rejects.toMatchObject({ status: 404 });
+
+    const text = await renderMetrics();
+    // Must be labeled by /b2b/status, not the full query string
+    expect(series(text, "provider_calls_total", { endpoint: "/b2b/status", outcome: "4xx" })).toBe(1);
+    expect(text).not.toContain("ord_xyz_123");
+  });
+
+  it("counts a 4xx failure as outcome 4xx", async () => {
+    linq.failNext("/b2b/offramp", {
+      kind: "refuse",
+      status: 400,
+      body: { message: "Invalid bank code" },
+    });
+
+    await expect(
+      linqCall("/b2b/offramp", {
+        method: "POST",
+        operation: "offramp_order",
+        body: {},
+      }),
+    ).rejects.toMatchObject({ status: 400 });
+
+    const text = await renderMetrics();
+    expect(series(text, "provider_calls_total", { endpoint: "/b2b/offramp", outcome: "4xx" })).toBe(1);
+    expect(sampleHistogram(text, "count", { provider: "linq", operation: "offramp_order", outcome: "4xx" })).toBe(1);
+  });
+
+  it("counts a 5xx failure as outcome 5xx", async () => {
+    linq.failNext("/b2b/onramp", {
+      kind: "refuse",
+      status: 500,
+      body: { message: "Internal server error" },
+    });
+
+    await expect(
+      linqCall("/b2b/onramp", {
+        method: "POST",
+        operation: "onramp_order",
+        body: {},
+      }),
+    ).rejects.toMatchObject({ status: 500 });
+
+    const text = await renderMetrics();
+    expect(series(text, "provider_calls_total", { endpoint: "/b2b/onramp", outcome: "5xx" })).toBe(1);
+    expect(sampleHistogram(text, "count", { provider: "linq", operation: "onramp_order", outcome: "5xx" })).toBe(1);
+  });
+
+  it("counts a network drop as outcome 5xx", async () => {
+    linq.failNext("/b2b/customers", {
+      kind: "drop-after-work",
+    });
+
+    await expect(
+      linqCall("/b2b/customers", {
+        method: "POST",
+        operation: "onramp_customer",
+        body: { customerRef: "c_drop_1" },
+      }),
+    ).rejects.toMatchObject({ status: 502 });
+
+    const text = await renderMetrics();
+    expect(series(text, "provider_calls_total", { endpoint: "/b2b/customers", outcome: "5xx" })).toBe(1);
+  });
+
+  it("counts a 429 rate limit as outcome 429", async () => {
+    linq.failNext("/b2b/bills/pay", {
+      kind: "refuse",
+      status: 429,
+      body: { message: "Rate limit exceeded" },
+    });
+
+    await expect(
+      linqCall("/b2b/bills/pay", {
+        method: "POST",
+        operation: "bill_pay",
+        body: {},
+      }),
+    ).rejects.toMatchObject({ status: 429 });
+
+    const text = await renderMetrics();
+    expect(series(text, "provider_calls_total", { endpoint: "/b2b/bills/pay", outcome: "429" })).toBe(1);
+    expect(sampleHistogram(text, "count", { provider: "linq", operation: "bill_pay", outcome: "429" })).toBe(1);
+  });
+
+  it("records a timeout as outcome timeout and records latency", async () => {
+    // deadline already expired
+    await expect(
+      linqCall("/b2b/offramp", {
+        operation: "offramp_order",
+        deadline: Date.now() - 10,
+      }),
+    ).rejects.toMatchObject({ status: 504, code: "provider_timeout" });
+
+    const text = await renderMetrics();
+    expect(series(text, "provider_calls_total", { endpoint: "/b2b/offramp", outcome: "timeout" })).toBe(1);
+    expect(sampleHistogram(text, "count", { provider: "linq", operation: "offramp_order", outcome: "timeout" })).toBe(1);
+    expect(sampleHistogram(text, "sum", { provider: "linq", operation: "offramp_order", outcome: "timeout" })).toBeGreaterThanOrEqual(0);
+  });
+
+  it("asserts label names are strictly bounded to endpoint and outcome", async () => {
+    await linqCall("/b2b/customers", {
+      method: "POST",
+      operation: "onramp_customer",
+      body: { customerRef: "c_bound_test" },
+    });
+
+    const text = await renderMetrics();
+    const metricLines = text.split("\n").filter((l) => l.startsWith("provider_calls_total{"));
+    expect(metricLines.length).toBeGreaterThan(0);
+    for (const line of metricLines) {
+      expect(line).toMatch(/^provider_calls_total\{endpoint="[^"]+",outcome="(success|4xx|5xx|429|timeout)"\} \d+$/);
+    }
+  });
+});

src/__tests__/linq/providerLatency.test.ts

@@ -99,8 +99,8 @@ describe("provider_request_duration_seconds", () => {
     await expect(createOfframpOrder(order("idem-fail-1"))).rejects.toMatchObject({ status: 400 });
 
     const text = await renderMetrics();
-    expect(sample(text, "count", { ...OFFRAMP_ORDER, outcome: "error" })).toBe(1);
-    expect(sample(text, "sum", { ...OFFRAMP_ORDER, outcome: "error" })).toBeGreaterThan(0);
+    expect(sample(text, "count", { ...OFFRAMP_ORDER, outcome: "4xx" })).toBe(1);
+    expect(sample(text, "sum", { ...OFFRAMP_ORDER, outcome: "4xx" })).toBeGreaterThan(0);
   });
 
   it("uses buckets that cover the 35-second provider budget", () => {

src/linq/http.ts

@@ -26,6 +26,7 @@
  */
 
 import {
+  providerCallsTotal,
   providerRateLimitTotal,
   providerRateLimitWaitSeconds,
   providerRequestDurationSeconds,
@@ -152,8 +153,37 @@ export type LinqOperation =
   | "bill_pay"
   | "bill_status";
 
-/** How a provider call ended, for the latency histogram's `outcome` label. */
-export type ProviderCallOutcome = "success" | "error" | "timeout";
+/** How a provider call ended, for `provider_calls_total` and latency metrics. */
+export type ProviderCallOutcome = "success" | "4xx" | "5xx" | "429" | "timeout";
+
+/**
+ * Normalise a provider path into a bounded endpoint path label.
+ *
+ * Drops query strings (e.g. `/b2b/status?id=...` -> `/b2b/status`) and ensures
+ * leading slash, so metrics labels stay strictly bounded.
+ */
+export function normalizeEndpointPath(path: string): string {
+  const queryIndex = path.indexOf("?");
+  const clean = queryIndex === -1 ? path : path.slice(0, queryIndex);
+  return clean.startsWith("/") ? clean : `/${clean}`;
+}
+
+/**
+ * Classify a failure into one of the bounded outcome categories:
+ * - "timeout": timeout or abort
+ * - "429": rate limited
+ * - "4xx": client-caused 4xx error (e.g. 400 bad request)
+ * - "5xx": provider-caused 5xx error or connection failure
+ */
+export function classifyFailureOutcome(err: unknown): "4xx" | "5xx" | "429" | "timeout" {
+  if (linqTransport(err) === "timeout") return "timeout";
+  if (err instanceof LinqError) {
+    if (err.status === 429) return "429";
+    if (err.status >= 400 && err.status < 500) return "4xx";
+    if (err.status >= 500) return "5xx";
+  }
+  return "5xx";
+}
 
 export interface LinqCallOptions {
   /**
@@ -454,9 +484,12 @@ export async function linqCall<T>(path: string, options: LinqCallOptions): Promi
       }
     }
   } catch (err) {
-    outcome = linqTransport(err) === "timeout" ? "timeout" : "error";
+    outcome = classifyFailureOutcome(err);
     throw err;
   } finally {
+    const endpoint = normalizeEndpointPath(path);
+    providerCallsTotal.inc({ endpoint, outcome });
+
     // Observed on failure too. A call that spent the full 35 seconds and then
     // failed is the one an operator most needs to see, and dropping it would
     // make this histogram describe only the healthy path.
@@ -474,7 +507,7 @@ export async function linqCall<T>(path: string, options: LinqCallOptions): Promi
       "linq",
       outcome === "success",
       Math.round(elapsedMs),
-      outcome === "success" ? undefined : outcome,
+      outcome === "timeout" ? "timeout" : outcome === "success" ? undefined : "error",
     );
   }
 }

src/metrics.ts

@@ -66,6 +66,23 @@ export const httpRequestDurationSeconds = new Histogram({
   registers: [registry],
 });
 
+/**
+ * Total payment-provider calls by endpoint path and outcome.
+ *
+ * Every naira operation is an HTTP call to Linq through src/linq/http.ts.
+ * This counter is incremented inside `linqCall` so no rail can forget to do it.
+ *
+ * Labels are bounded:
+ *  - `endpoint`: the normalized endpoint path (e.g. "/b2b/offramp", "/b2b/status")
+ *  - `outcome`: one of "success", "4xx", "5xx", "429", or "timeout"
+ */
+export const providerCallsTotal = new Counter({
+  name: "provider_calls_total",
+  help: "Total payment-provider calls by endpoint path and outcome",
+  labelNames: ["endpoint", "outcome"] as const,
+  registers: [registry],
+});
+
 /**
  * Time spent inside a payment-provider call, separately from the request.
  *