diff --git a/src/__tests__/linq/providerCalls.test.ts b/src/__tests__/linq/providerCalls.test.ts new file mode 100644 index 00000000..f81d65d6 --- /dev/null +++ b/src/__tests__/linq/providerCalls.test.ts @@ -0,0 +1,197 @@ +/** + * Tests for provider_calls_total and Linq call outcome metrics. + * + * Requirements: + * - Counter of provider calls by endpoint path and outcome, incremented in src/linq/http.ts + * - Every call through linqCall is counted, with no per-rail opt-in + * - Labels are bounded: endpoint path and outcome only + * - Outcome distinguishes: success, 4xx, 5xx, 429, timeout + * - Latency is observed for failures as well as successes + * - Timeout is recorded as timeout, not generic failure + * - Tested against fake provider (never real Linq) + */ +import { linqCall } from "../../linq/http"; +import { renderMetrics, _resetMetrics } from "../../metrics"; +import { installFakeLinq, type FakeLinq } from "../fakes/linqProvider"; + +process.env.LINQ_API_KEY = "biz_live_test"; + +let linq: FakeLinq; + +beforeEach(() => { + _resetMetrics(); + linq = installFakeLinq(); + jest.spyOn(console, "warn").mockImplementation(() => {}); +}); + +afterEach(() => { + linq.restore(); + jest.restoreAllMocks(); +}); + +function series( + text: string, + metric: string, + labels: Record, +): number | undefined { + const line = text.split("\n").find( + (candidate) => + candidate.startsWith(`${metric}{`) && + Object.entries(labels).every(([key, value]) => candidate.includes(`${key}="${value}"`)), + ); + return line === undefined ? undefined : Number(line.slice(line.lastIndexOf(" ") + 1)); +} + +function sampleHistogram( + text: string, + suffix: "count" | "sum", + labels: Record, +): number | undefined { + const line = text.split("\n").find( + (candidate) => + candidate.startsWith(`provider_request_duration_seconds_${suffix}{`) && + Object.entries(labels).every(([key, value]) => candidate.includes(`${key}="${value}"`)), + ); + return line === undefined ? undefined : Number(line.slice(line.lastIndexOf(" ") + 1)); +} + +describe("provider_calls_total", () => { + it("counts a successful call by endpoint path and outcome", async () => { + await linqCall("/b2b/customers", { + method: "POST", + operation: "onramp_customer", + body: { customerRef: "c_test_1" }, + }); + + const text = await renderMetrics(); + expect(series(text, "provider_calls_total", { endpoint: "/b2b/customers", outcome: "success" })).toBe(1); + expect(sampleHistogram(text, "count", { provider: "linq", operation: "onramp_customer", outcome: "success" })).toBe(1); + }); + + it("normalizes paths by stripping query parameters to preserve bounded cardinality", async () => { + // Calling an endpoint with dynamic query parameters + linq.failNext("/b2b/status", { + kind: "refuse", + status: 404, + body: { message: "Order not found" }, + }); + + await expect( + linqCall("/b2b/status?id=ord_xyz_123&foo=bar", { + operation: "offramp_status", + }), + ).rejects.toMatchObject({ status: 404 }); + + const text = await renderMetrics(); + // Must be labeled by /b2b/status, not the full query string + expect(series(text, "provider_calls_total", { endpoint: "/b2b/status", outcome: "4xx" })).toBe(1); + expect(text).not.toContain("ord_xyz_123"); + }); + + it("counts a 4xx failure as outcome 4xx", async () => { + linq.failNext("/b2b/offramp", { + kind: "refuse", + status: 400, + body: { message: "Invalid bank code" }, + }); + + await expect( + linqCall("/b2b/offramp", { + method: "POST", + operation: "offramp_order", + body: {}, + }), + ).rejects.toMatchObject({ status: 400 }); + + const text = await renderMetrics(); + expect(series(text, "provider_calls_total", { endpoint: "/b2b/offramp", outcome: "4xx" })).toBe(1); + expect(sampleHistogram(text, "count", { provider: "linq", operation: "offramp_order", outcome: "4xx" })).toBe(1); + }); + + it("counts a 5xx failure as outcome 5xx", async () => { + linq.failNext("/b2b/onramp", { + kind: "refuse", + status: 500, + body: { message: "Internal server error" }, + }); + + await expect( + linqCall("/b2b/onramp", { + method: "POST", + operation: "onramp_order", + body: {}, + }), + ).rejects.toMatchObject({ status: 500 }); + + const text = await renderMetrics(); + expect(series(text, "provider_calls_total", { endpoint: "/b2b/onramp", outcome: "5xx" })).toBe(1); + expect(sampleHistogram(text, "count", { provider: "linq", operation: "onramp_order", outcome: "5xx" })).toBe(1); + }); + + it("counts a network drop as outcome 5xx", async () => { + linq.failNext("/b2b/customers", { + kind: "drop-after-work", + }); + + await expect( + linqCall("/b2b/customers", { + method: "POST", + operation: "onramp_customer", + body: { customerRef: "c_drop_1" }, + }), + ).rejects.toMatchObject({ status: 502 }); + + const text = await renderMetrics(); + expect(series(text, "provider_calls_total", { endpoint: "/b2b/customers", outcome: "5xx" })).toBe(1); + }); + + it("counts a 429 rate limit as outcome 429", async () => { + linq.failNext("/b2b/bills/pay", { + kind: "refuse", + status: 429, + body: { message: "Rate limit exceeded" }, + }); + + await expect( + linqCall("/b2b/bills/pay", { + method: "POST", + operation: "bill_pay", + body: {}, + }), + ).rejects.toMatchObject({ status: 429 }); + + const text = await renderMetrics(); + expect(series(text, "provider_calls_total", { endpoint: "/b2b/bills/pay", outcome: "429" })).toBe(1); + expect(sampleHistogram(text, "count", { provider: "linq", operation: "bill_pay", outcome: "429" })).toBe(1); + }); + + it("records a timeout as outcome timeout and records latency", async () => { + // deadline already expired + await expect( + linqCall("/b2b/offramp", { + operation: "offramp_order", + deadline: Date.now() - 10, + }), + ).rejects.toMatchObject({ status: 504, code: "provider_timeout" }); + + const text = await renderMetrics(); + expect(series(text, "provider_calls_total", { endpoint: "/b2b/offramp", outcome: "timeout" })).toBe(1); + expect(sampleHistogram(text, "count", { provider: "linq", operation: "offramp_order", outcome: "timeout" })).toBe(1); + expect(sampleHistogram(text, "sum", { provider: "linq", operation: "offramp_order", outcome: "timeout" })).toBeGreaterThanOrEqual(0); + }); + + it("asserts label names are strictly bounded to endpoint and outcome", async () => { + await linqCall("/b2b/customers", { + method: "POST", + operation: "onramp_customer", + body: { customerRef: "c_bound_test" }, + }); + + const text = await renderMetrics(); + const metricLines = text.split("\n").filter((l) => l.startsWith("provider_calls_total{")); + expect(metricLines.length).toBeGreaterThan(0); + for (const line of metricLines) { + expect(line).toMatch(/^provider_calls_total\{endpoint="[^"]+",outcome="(success|4xx|5xx|429|timeout)"\} \d+$/); + } + }); +}); diff --git a/src/__tests__/linq/providerLatency.test.ts b/src/__tests__/linq/providerLatency.test.ts index dbf88954..48b86c60 100644 --- a/src/__tests__/linq/providerLatency.test.ts +++ b/src/__tests__/linq/providerLatency.test.ts @@ -99,8 +99,8 @@ describe("provider_request_duration_seconds", () => { await expect(createOfframpOrder(order("idem-fail-1"))).rejects.toMatchObject({ status: 400 }); const text = await renderMetrics(); - expect(sample(text, "count", { ...OFFRAMP_ORDER, outcome: "error" })).toBe(1); - expect(sample(text, "sum", { ...OFFRAMP_ORDER, outcome: "error" })).toBeGreaterThan(0); + expect(sample(text, "count", { ...OFFRAMP_ORDER, outcome: "4xx" })).toBe(1); + expect(sample(text, "sum", { ...OFFRAMP_ORDER, outcome: "4xx" })).toBeGreaterThan(0); }); it("uses buckets that cover the 35-second provider budget", () => { diff --git a/src/linq/http.ts b/src/linq/http.ts index 1e51f296..96f9fef5 100644 --- a/src/linq/http.ts +++ b/src/linq/http.ts @@ -26,6 +26,7 @@ */ import { + providerCallsTotal, providerRateLimitTotal, providerRateLimitWaitSeconds, providerRequestDurationSeconds, @@ -152,8 +153,37 @@ export type LinqOperation = | "bill_pay" | "bill_status"; -/** How a provider call ended, for the latency histogram's `outcome` label. */ -export type ProviderCallOutcome = "success" | "error" | "timeout"; +/** How a provider call ended, for `provider_calls_total` and latency metrics. */ +export type ProviderCallOutcome = "success" | "4xx" | "5xx" | "429" | "timeout"; + +/** + * Normalise a provider path into a bounded endpoint path label. + * + * Drops query strings (e.g. `/b2b/status?id=...` -> `/b2b/status`) and ensures + * leading slash, so metrics labels stay strictly bounded. + */ +export function normalizeEndpointPath(path: string): string { + const queryIndex = path.indexOf("?"); + const clean = queryIndex === -1 ? path : path.slice(0, queryIndex); + return clean.startsWith("/") ? clean : `/${clean}`; +} + +/** + * Classify a failure into one of the bounded outcome categories: + * - "timeout": timeout or abort + * - "429": rate limited + * - "4xx": client-caused 4xx error (e.g. 400 bad request) + * - "5xx": provider-caused 5xx error or connection failure + */ +export function classifyFailureOutcome(err: unknown): "4xx" | "5xx" | "429" | "timeout" { + if (linqTransport(err) === "timeout") return "timeout"; + if (err instanceof LinqError) { + if (err.status === 429) return "429"; + if (err.status >= 400 && err.status < 500) return "4xx"; + if (err.status >= 500) return "5xx"; + } + return "5xx"; +} export interface LinqCallOptions { /** @@ -454,9 +484,12 @@ export async function linqCall(path: string, options: LinqCallOptions): Promi } } } catch (err) { - outcome = linqTransport(err) === "timeout" ? "timeout" : "error"; + outcome = classifyFailureOutcome(err); throw err; } finally { + const endpoint = normalizeEndpointPath(path); + providerCallsTotal.inc({ endpoint, outcome }); + // Observed on failure too. A call that spent the full 35 seconds and then // failed is the one an operator most needs to see, and dropping it would // make this histogram describe only the healthy path. @@ -474,7 +507,7 @@ export async function linqCall(path: string, options: LinqCallOptions): Promi "linq", outcome === "success", Math.round(elapsedMs), - outcome === "success" ? undefined : outcome, + outcome === "timeout" ? "timeout" : outcome === "success" ? undefined : "error", ); } } diff --git a/src/metrics.ts b/src/metrics.ts index 4e82003e..371a1813 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -66,6 +66,23 @@ export const httpRequestDurationSeconds = new Histogram({ registers: [registry], }); +/** + * Total payment-provider calls by endpoint path and outcome. + * + * Every naira operation is an HTTP call to Linq through src/linq/http.ts. + * This counter is incremented inside `linqCall` so no rail can forget to do it. + * + * Labels are bounded: + * - `endpoint`: the normalized endpoint path (e.g. "/b2b/offramp", "/b2b/status") + * - `outcome`: one of "success", "4xx", "5xx", "429", or "timeout" + */ +export const providerCallsTotal = new Counter({ + name: "provider_calls_total", + help: "Total payment-provider calls by endpoint path and outcome", + labelNames: ["endpoint", "outcome"] as const, + registers: [registry], +}); + /** * Time spent inside a payment-provider call, separately from the request. *