From 406dc05f57981e700db0e0fa6010776ddedf8199 Mon Sep 17 00:00:00 2001
From: Aboyeji Isaac <aboyejiisaac@gmail.com>
Date: Tue, 20 Oct 2026 12:10:00 +0100
Subject: [PATCH 01/37] Made a little harmless changes

---
 .commit_message.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.commit_message.txt b/.commit_message.txt
index cd3b5d9..27f0bef 100644
--- a/.commit_message.txt
+++ b/.commit_message.txt
@@ -1,5 +1,7 @@
 feat(#112): Implement Socket.IO load tests for 10,000 concurrent connections
 
+
+
 - Add metrics collection service (/api/v1/socket-metrics) with real-time latency tracking
 - Create k6 load test scenario with 10k VU ramp profile and latency assertions
 - Implement optimized data seeding scripts (default 25/25/50, 10k variants)

From 872a2f6a65f410c0bda57bfbf258c5367c7edba9 Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:45:21 +0100
Subject: [PATCH 02/37] feat(metrics): add Prometheus metrics collector

---
 src/services/prometheusMetricsService.ts | 221 +++++++++++++++++++++++
 1 file changed, 221 insertions(+)
 create mode 100644 src/services/prometheusMetricsService.ts

diff --git a/src/services/prometheusMetricsService.ts b/src/services/prometheusMetricsService.ts
new file mode 100644
index 0000000..ff67ff7
--- /dev/null
+++ b/src/services/prometheusMetricsService.ts
@@ -0,0 +1,221 @@
+import mongoose from 'mongoose';
+import { redisClient } from '../config/redis';
+import { getAllCircuitBreakerStatuses } from '../utils/circuitBreaker';
+import socketMetricsService from './socketMetricsService';
+import { getIndexerLagSnapshot } from './monitorService';
+
+const LATENCY_BUCKETS_SECONDS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10];
+
+interface HistogramSnapshot {
+  count: number;
+  sumSeconds: number;
+  bucketCounts: number[];
+}
+
+interface MongoPoolSnapshot {
+  total: number;
+  available: number;
+  introspectionAvailable: boolean;
+}
+
+const escapeLabel = (value: string): string =>
+  value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n');
+
+const labelsToString = (labels: Record<string, string | number>): string =>
+  '{' + Object.entries(labels).map(([key, value]) => key + '="' + escapeLabel(String(value)) + '"').join(',') + '}';
+
+const safeMetricLabel = (value: string): string =>
+  value.replace(/[^a-zA-Z0-9_.-]/g, '_').slice(0, 100) || 'unknown';
+
+const readMongoPoolSnapshot = (): MongoPoolSnapshot => {
+  const connection = mongoose.connection;
+  const topology = (connection.getClient() as unknown as {
+    topology?: {
+      s?: {
+        servers?: Map<string, {
+          s?: {
+            pool?: {
+              totalConnectionCount?: number;
+              availableConnectionCount?: number;
+            };
+          };
+        }>;
+      };
+    };
+  }).topology;
+
+  const servers = topology?.s?.servers;
+  if (!servers || typeof servers.values !== 'function') {
+    return { total: 0, available: 0, introspectionAvailable: false };
+  }
+
+  let total = 0;
+  let available = 0;
+  for (const server of servers.values()) {
+    const pool = server.s?.pool;
+    if (!pool) continue;
+    total += pool.totalConnectionCount ?? 0;
+    available += pool.availableConnectionCount ?? 0;
+  }
+
+  return { total, available, introspectionAvailable: true };
+};
+
+/**
+ * Collects process-local HTTP metrics and snapshots of existing operational
+ * dependencies. Request labels use Express route templates rather than raw
+ * URLs so identifiers and arbitrary query strings never become metric labels.
+ */
+class PrometheusMetricsService {
+  private readonly requestCounters = new Map<string, number>();
+  private readonly requestHistograms = new Map<string, HistogramSnapshot>();
+
+  public recordRequest(method: string, route: string, statusCode: number, durationMs: number): void {
+    const normalizedMethod = method.toUpperCase();
+    const normalizedRoute = route || 'unmatched';
+    const counterKey = JSON.stringify([normalizedMethod, normalizedRoute, statusCode]);
+    this.requestCounters.set(counterKey, (this.requestCounters.get(counterKey) ?? 0) + 1);
+
+    const histogramKey = JSON.stringify([normalizedMethod, normalizedRoute]);
+    let histogram = this.requestHistograms.get(histogramKey);
+    if (!histogram) {
+      histogram = {
+        count: 0,
+        sumSeconds: 0,
+        bucketCounts: LATENCY_BUCKETS_SECONDS.map(() => 0),
+      };
+      this.requestHistograms.set(histogramKey, histogram);
+    }
+
+    const durationSeconds = Math.max(0, durationMs) / 1000;
+    histogram.count += 1;
+    histogram.sumSeconds += durationSeconds;
+    LATENCY_BUCKETS_SECONDS.forEach((boundary, index) => {
+      if (durationSeconds <= boundary) histogram!.bucketCounts[index] += 1;
+    });
+  }
+
+  public async render(): Promise<string> {
+    const lines: string[] = [];
+    const gauge = (name: string, help: string, value: number): void => {
+      lines.push(
+        '# HELP ' + name + ' ' + help,
+        '# TYPE ' + name + ' gauge',
+        name + ' ' + (Number.isFinite(value) ? value : 0),
+      );
+    };
+
+    lines.push(
+      '# HELP swiftchain_http_requests_total Total HTTP requests served by route.',
+      '# TYPE swiftchain_http_requests_total counter',
+    );
+    for (const [key, value] of this.requestCounters) {
+      const [method, route, statusCode] = JSON.parse(key) as [string, string, number];
+      lines.push(
+        'swiftchain_http_requests_total' +
+          labelsToString({ method, route, status_code: statusCode }) +
+          ' ' + value,
+      );
+    }
+
+    lines.push(
+      '# HELP swiftchain_http_request_duration_seconds HTTP request duration in seconds.',
+      '# TYPE swiftchain_http_request_duration_seconds histogram',
+    );
+    for (const [key, histogram] of this.requestHistograms) {
+      const [method, route] = JSON.parse(key) as [string, string];
+      LATENCY_BUCKETS_SECONDS.forEach((boundary, index) => {
+        lines.push(
+          'swiftchain_http_request_duration_seconds_bucket' +
+            labelsToString({ method, route, le: boundary }) +
+            ' ' + histogram.bucketCounts[index],
+        );
+      });
+      lines.push(
+        'swiftchain_http_request_duration_seconds_bucket' +
+          labelsToString({ method, route, le: '+Inf' }) + ' ' + histogram.count,
+        'swiftchain_http_request_duration_seconds_sum' +
+          labelsToString({ method, route }) + ' ' + histogram.sumSeconds,
+        'swiftchain_http_request_duration_seconds_count' +
+          labelsToString({ method, route }) + ' ' + histogram.count,
+      );
+    }
+
+    gauge('swiftchain_process_uptime_seconds', 'Process uptime in seconds.', process.uptime());
+    const memory = process.memoryUsage();
+    gauge('swiftchain_process_resident_memory_bytes', 'Resident process memory in bytes.', memory.rss);
+    gauge('swiftchain_process_heap_used_bytes', 'Node.js heap memory used in bytes.', memory.heapUsed);
+    gauge('swiftchain_process_heap_total_bytes', 'Node.js heap memory allocated in bytes.', memory.heapTotal);
+
+    const mongoPool = readMongoPoolSnapshot();
+    gauge(
+      'swiftchain_mongodb_connection_ready',
+      'Whether the Mongoose connection is connected.',
+      mongoose.connection.readyState === 1 ? 1 : 0,
+    );
+    gauge(
+      'swiftchain_mongodb_pool_stats_available',
+      'Whether MongoDB driver pool statistics could be read from the active topology.',
+      mongoPool.introspectionAvailable ? 1 : 0,
+    );
+    lines.push(
+      '# HELP swiftchain_mongodb_pool_connections MongoDB pool connections by state.',
+      '# TYPE swiftchain_mongodb_pool_connections gauge',
+      'swiftchain_mongodb_pool_connections' + labelsToString({ state: 'total' }) + ' ' + mongoPool.total,
+      'swiftchain_mongodb_pool_connections' + labelsToString({ state: 'available' }) + ' ' + mongoPool.available,
+      'swiftchain_mongodb_pool_connections' +
+        labelsToString({ state: 'in_use' }) + ' ' + Math.max(0, mongoPool.total - mongoPool.available),
+    );
+
+    let redisHealthy = 0;
+    if (redisClient.status === 'ready') {
+      try {
+        redisHealthy = (await redisClient.ping()) === 'PONG' ? 1 : 0;
+      } catch {
+        redisHealthy = 0;
+      }
+    }
+    gauge('swiftchain_redis_healthy', 'Whether Redis responds to a PING command.', redisHealthy);
+
+    const breakers = getAllCircuitBreakerStatuses();
+    lines.push(
+      '# HELP swiftchain_circuit_breaker_open Whether a named circuit breaker is open or half-open.',
+      '# TYPE swiftchain_circuit_breaker_open gauge',
+    );
+    for (const breaker of breakers) {
+      lines.push(
+        'swiftchain_circuit_breaker_open' +
+          labelsToString({ name: safeMetricLabel(breaker.name), state: breaker.state }) +
+          ' ' + (breaker.state === 'closed' ? 0 : 1),
+      );
+    }
+
+    try {
+      const lag = await getIndexerLagSnapshot();
+      gauge('swiftchain_indexer_lag_available', 'Whether indexer lag could be measured.', lag ? 1 : 0);
+      if (lag) {
+        gauge(
+          'swiftchain_indexer_lag_ledgers',
+          'Number of ledgers the indexer trails the network by.',
+          lag.lagLedgers,
+        );
+      }
+    } catch {
+      gauge('swiftchain_indexer_lag_available', 'Whether indexer lag could be measured.', 0);
+    }
+
+    const socketMetrics = socketMetricsService.getMetrics();
+    gauge('swiftchain_socket_connected_sockets', 'Currently connected Socket.IO clients.', socketMetrics.connectedSockets);
+    gauge('swiftchain_socket_total_connections', 'Total Socket.IO connections since process start.', socketMetrics.totalConnections);
+    gauge('swiftchain_socket_total_disconnections', 'Total Socket.IO disconnections since process start.', socketMetrics.totalDisconnections);
+    gauge('swiftchain_socket_messages_processed', 'Total Socket.IO messages processed since process start.', socketMetrics.messagesProcessed);
+    gauge('swiftchain_socket_message_latency_p50_ms', 'Socket.IO message latency p50 in milliseconds.', socketMetrics.messageLatencyMs.p50);
+    gauge('swiftchain_socket_message_latency_p95_ms', 'Socket.IO message latency p95 in milliseconds.', socketMetrics.messageLatencyMs.p95);
+    gauge('swiftchain_socket_message_latency_p99_ms', 'Socket.IO message latency p99 in milliseconds.', socketMetrics.messageLatencyMs.p99);
+
+    return lines.join('\n') + '\n';
+  }
+}
+
+export const prometheusMetricsService = new PrometheusMetricsService();
+export default prometheusMetricsService;

From eea14ab7d5341c074982550b89f310356705e58b Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:45:31 +0100
Subject: [PATCH 03/37] feat(metrics): expose read-only indexer lag snapshot

---
 src/services/monitorService.ts | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/src/services/monitorService.ts b/src/services/monitorService.ts
index 5418c0c..ca34728 100644
--- a/src/services/monitorService.ts
+++ b/src/services/monitorService.ts
@@ -161,6 +161,34 @@ export const checkIndexerLag = async (): Promise<IndexerLagCheckResult> => {
   return result;
 };
 
+/**
+ * Read-only indexer lag snapshot for scrape-based monitoring.
+ *
+ * Unlike checkIndexerLag(), this helper does not create checkpoints, persist
+ * alerts, or send webhooks, so frequent metrics scrapes have no write side effects.
+ */
+export interface IndexerLagSnapshot {
+  network: string;
+  processedLedger: number;
+  networkLedger: number;
+  lagLedgers: number;
+}
+
+export const getIndexerLagSnapshot = async (): Promise<IndexerLagSnapshot | null> => {
+  const status = await IndexerStatus.findOne({ network: stellarConfig.network });
+  if (!status) {
+    return null;
+  }
+
+  const networkLedger = await sorobanService.getLatestLedger();
+  return {
+    network: status.network,
+    processedLedger: status.lastProcessedLedger,
+    networkLedger,
+    lagLedgers: Math.max(0, networkLedger - status.lastProcessedLedger),
+  };
+};
+
 // ─── Alert history ─────────────────────────────────────────────────────────────
 
 /**

From 4fe40f54fce0e1cd1fb74e9731bdae13ce82c76e Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:45:39 +0100
Subject: [PATCH 04/37] feat(metrics): record HTTP request metrics

---
 src/middleware/prometheusMetricsMiddleware.ts | 31 +++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 src/middleware/prometheusMetricsMiddleware.ts

diff --git a/src/middleware/prometheusMetricsMiddleware.ts b/src/middleware/prometheusMetricsMiddleware.ts
new file mode 100644
index 0000000..09aa10a
--- /dev/null
+++ b/src/middleware/prometheusMetricsMiddleware.ts
@@ -0,0 +1,31 @@
+import { Request, Response, NextFunction } from 'express';
+import prometheusMetricsService from '../services/prometheusMetricsService';
+
+/**
+ * Records completed HTTP requests using route templates to avoid unbounded
+ * cardinality from identifiers and query parameters.
+ */
+const prometheusMetricsMiddleware = (
+  req: Request,
+  res: Response,
+  next: NextFunction,
+): void => {
+  const path = req.path;
+  if (path === '/metrics' || path === '/api/v1/metrics' || path === '/api/v1/metrics/') {
+    next();
+    return;
+  }
+
+  const startedAt = process.hrtime.bigint();
+  res.once('finish', () => {
+    const durationMs = Number(process.hrtime.bigint() - startedAt) / 1_000_000;
+    const routePath = req.route && typeof req.route.path === 'string'
+      ? (req.baseUrl + req.route.path) || req.route.path
+      : 'unmatched';
+    prometheusMetricsService.recordRequest(req.method, routePath, res.statusCode, durationMs);
+  });
+
+  next();
+};
+
+export default prometheusMetricsMiddleware;

From a4777f223ce161b3b4aa07a0b78431708a5e0cde Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:45:50 +0100
Subject: [PATCH 05/37] feat(metrics): add Prometheus metrics controller

---
 src/controllers/metricsController.ts | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 src/controllers/metricsController.ts

diff --git a/src/controllers/metricsController.ts b/src/controllers/metricsController.ts
new file mode 100644
index 0000000..cad0d22
--- /dev/null
+++ b/src/controllers/metricsController.ts
@@ -0,0 +1,23 @@
+import { Request, Response, NextFunction } from 'express';
+import { StatusCodes } from 'http-status-codes';
+import prometheusMetricsService from '../services/prometheusMetricsService';
+
+/**
+ * Exposes operational metrics in Prometheus text exposition format.
+ * Authentication and feature gating are enforced by metricsRoutes.
+ */
+export const getPrometheusMetrics = async (
+  _req: Request,
+  res: Response,
+  next: NextFunction,
+): Promise<void> => {
+  try {
+    const output = await prometheusMetricsService.render();
+    res
+      .status(StatusCodes.OK)
+      .setHeader('Content-Type', 'text/plain; version=0.0.4; charset=utf-8');
+    res.send(output);
+  } catch (error) {
+    next(error);
+  }
+};

From a601585061c790dee0764fd337b47319b9364cc6 Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:45:52 +0100
Subject: [PATCH 06/37] feat(metrics): protect Prometheus metrics route

---
 src/routes/metricsRoutes.ts | 46 +++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 src/routes/metricsRoutes.ts

diff --git a/src/routes/metricsRoutes.ts b/src/routes/metricsRoutes.ts
new file mode 100644
index 0000000..6ee4056
--- /dev/null
+++ b/src/routes/metricsRoutes.ts
@@ -0,0 +1,46 @@
+import { Router, Response, NextFunction, Request } from 'express';
+import authenticate from '../middleware/authenticate';
+import requireRole from '../middleware/requireRole';
+import { UserRole } from '../interfaces/IUser';
+import env from '../config/env';
+import { getPrometheusMetrics } from '../controllers/metricsController';
+
+const router = Router();
+
+/**
+ * @openapi
+ * /v1/metrics:
+ *   get:
+ *     tags: [Monitoring]
+ *     summary: Export operational metrics in Prometheus format
+ *     description: Admin-only endpoint. Disabled unless ENABLE_METRICS=true.
+ *     security:
+ *       - bearerAuth: []
+ *     responses:
+ *       200:
+ *         description: Prometheus text exposition
+ *         content:
+ *           text/plain:
+ *             schema:
+ *               type: string
+ *       401:
+ *         description: Authentication required
+ *       403:
+ *         description: Admin role required
+ *       404:
+ *         description: Metrics endpoint is disabled
+ */
+const requireMetricsEnabled = (_req: Request, res: Response, next: NextFunction): void => {
+  if (!env.ENABLE_METRICS) {
+    res.status(404).end();
+    return;
+  }
+  next();
+};
+
+router.use(requireMetricsEnabled);
+router.use(authenticate);
+router.use(requireRole(UserRole.ADMIN));
+router.get('/', getPrometheusMetrics);
+
+export default router;

From 8ece1e583618affee9c48790a8d1e50a26687c9b Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:45:59 +0100
Subject: [PATCH 07/37] feat(metrics): register versioned metrics route

---
 src/routes/index.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/routes/index.ts b/src/routes/index.ts
index d866373..a0dc123 100644
--- a/src/routes/index.ts
+++ b/src/routes/index.ts
@@ -10,6 +10,7 @@ import fleetRoutes from './fleetRoutes';
 import disputeRoutes from './disputeRoutes';
 import eventLogRoutes from './eventLogRoutes';
 import monitorRoutes from './monitorRoutes';
+import metricsRoutes from './metricsRoutes';
 import profileRoutes from './profileRoutes';
 import notificationRoutes from './notificationRoutes';
 import healthRoutes from './healthRoutes';
@@ -43,6 +44,7 @@ router.use('/v1/fleets', fleetRoutes);
 router.use('/v1/disputes', disputeRoutes);
 router.use('/v1/eventlog', eventLogRoutes);
 router.use('/v1/monitor', monitorRoutes);
+router.use('/v1/metrics', metricsRoutes);
 router.use('/v1/profile', profileRoutes);
 router.use('/v1/notifications', notificationRoutes);
 router.use('/v1/health', healthRoutes);

From 5a8ff22f1a93f193eb3e2786a0179b708f429413 Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:46:06 +0100
Subject: [PATCH 08/37] feat(metrics): add validated metrics feature flag

---
 src/config/env.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/config/env.ts b/src/config/env.ts
index a542efb..ec6fe8a 100644
--- a/src/config/env.ts
+++ b/src/config/env.ts
@@ -18,6 +18,7 @@ interface EnvConfig {
   CORS_ORIGIN: string;
   RATE_LIMIT_WINDOW_MS: number;
   RATE_LIMIT_MAX_REQUESTS: number;
+  ENABLE_METRICS: boolean;
   DISPUTE_NOTIFICATION_WEBHOOK_URL: string;
   UPLOAD_STORAGE_DRIVER: string;
   UPLOAD_LOCAL_DIR: string;
@@ -229,6 +230,7 @@ const envSchema = z.object({
   CORS_ORIGIN: z.string().default('*'),
   RATE_LIMIT_WINDOW_MS: numeric(z.coerce.number().int().min(1000).default(900000)),
   RATE_LIMIT_MAX_REQUESTS: numeric(z.coerce.number().int().min(1).default(100)),
+  ENABLE_METRICS: z.preprocess(blankToUndefined, z.enum(['true', 'false']).default('false').transform((value) => value === 'true')),
   DISPUTE_NOTIFICATION_WEBHOOK_URL: z.string().default(''),
 
   // ── Uploads / storage ──────────────────────────────────────────────────────

From ae50e03e589fe955b48edb9325cd0dfe225f6419 Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:46:13 +0100
Subject: [PATCH 09/37] docs(metrics): document ENABLE_METRICS configuration

---
 .env.example | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.env.example b/.env.example
index 3f957cc..920a513 100644
--- a/.env.example
+++ b/.env.example
@@ -73,6 +73,9 @@ RATE_LIMIT_WINDOW_MS=900000
 # Maximum requests per client per window before a 429 is returned. [default: 100]
 RATE_LIMIT_MAX_REQUESTS=100
 
+## Prometheus metrics endpoint (disabled by default; admin JWT required)
+ENABLE_METRICS=false
+
 # Outbound webhook that receives dispute lifecycle notifications. Empty string
 # (the default) disables notifications. [default: "" — set to enable]
 # NOTE: declared in the schema but not yet read by any service; wire it up in

From 44334bc23694e49de748bd0fa21c199bf9f3e1db Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:46:18 +0100
Subject: [PATCH 10/37] feat(metrics): mount protected Prometheus endpoint

---
 src/app.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/app.ts b/src/app.ts
index eabf7fa..784b877 100644
--- a/src/app.ts
+++ b/src/app.ts
@@ -8,6 +8,8 @@ import rateLimit from 'express-rate-limit';
 import swaggerUi from 'swagger-ui-express';
 
 import routes from './routes';
+import metricsRoutes from './routes/metricsRoutes';
+import prometheusMetricsMiddleware from './middleware/prometheusMetricsMiddleware';
 import logger from './config/logger';
 import { sendError } from './utils/responseWrapper';
 import { connectDatabase } from './config/database';
@@ -37,6 +39,7 @@ app.use(compression());
 // Track in-flight requests and reject new ones during graceful shutdown.
 app.use(requestTracker);
 app.use(requestLogger);
+app.use(prometheusMetricsMiddleware);
 
 // Swagger UI needs inline <script>/<style>, which the default Helmet CSP
 // blocks, so it gets its own relaxed CSP scoped to /api-docs only — the
@@ -91,6 +94,9 @@ app.get('/health', (_req, res): void => {
   });
 });
 
+// Prometheus scrape path follows the conventional /metrics URL; the same
+// protected endpoint is also available at /api/v1/metrics.
+app.use('/metrics', metricsRoutes);
 app.use('/api', routes);
 
 app.use((req, res): void => {

From 5e4e9d5986d19abf2d2dddf34767b3b01930af92 Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:46:43 +0100
Subject: [PATCH 11/37] docs(metrics): document Prometheus endpoint and alert
 guidance

---
 docs/prometheus-metrics.md | 62 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)
 create mode 100644 docs/prometheus-metrics.md

diff --git a/docs/prometheus-metrics.md b/docs/prometheus-metrics.md
new file mode 100644
index 0000000..0582bf0
--- /dev/null
+++ b/docs/prometheus-metrics.md
@@ -0,0 +1,62 @@
+# Prometheus metrics
+
+SwiftChain Backend exposes a Prometheus text exposition endpoint for operational monitoring.
+
+## Enable and access
+
+Set this validated environment variable in .env:
+
+    ENABLE_METRICS=true
+
+The endpoint is disabled by default. When disabled, both paths return 404:
+
+- GET /metrics
+- GET /api/v1/metrics
+
+When enabled, both require a valid Bearer JWT for a user with the admin role. Keep the endpoint behind TLS and an operator-only network boundary; do not expose admin credentials in Prometheus scrape configuration. Use a dedicated monitoring identity and rotate its token according to your security policy.
+
+Example request:
+
+    curl -H "Authorization: Bearer $METRICS_ADMIN_TOKEN" http://localhost:3000/metrics
+
+The response uses text/plain; version=0.0.4; charset=utf-8 and can be scraped by Prometheus-compatible collectors.
+
+## Exported metrics
+
+| Metric | Type | Meaning |
+| --- | --- | --- |
+| swiftchain_http_requests_total | Counter | Completed HTTP requests, labelled by method, Express route template, and status code |
+| swiftchain_http_request_duration_seconds | Histogram | Request duration buckets, sum, and count |
+| swiftchain_process_uptime_seconds | Gauge | Process uptime |
+| swiftchain_process_resident_memory_bytes | Gauge | Resident memory |
+| swiftchain_process_heap_used_bytes / swiftchain_process_heap_total_bytes | Gauge | Node.js heap usage |
+| swiftchain_mongodb_connection_ready | Gauge | Whether Mongoose is connected |
+| swiftchain_mongodb_pool_stats_available | Gauge | Whether the installed MongoDB driver exposes pool statistics |
+| swiftchain_mongodb_pool_connections | Gauge | Total, available, and in-use MongoDB pool connections |
+| swiftchain_redis_healthy | Gauge | Result of a Redis PING check |
+| swiftchain_circuit_breaker_open | Gauge | Circuit-breaker state by registered breaker |
+| swiftchain_indexer_lag_available | Gauge | Whether a lag snapshot could be read |
+| swiftchain_indexer_lag_ledgers | Gauge | Gap between persisted processed ledger and the live network ledger |
+| swiftchain_socket_* | Gauge | Existing Socket.IO connection, message, and latency snapshots |
+
+HTTP route labels use Express route templates rather than raw request URLs, preventing user IDs, arbitrary paths, and query values from creating high-cardinality labels. The indexer metric reads the current checkpoint and live ledger without persisting alerts or sending webhooks during a scrape.
+
+MongoDB pool metrics depend on runtime pool fields exposed by the installed MongoDB driver. Check swiftchain_mongodb_pool_stats_available before relying on pool connection values.
+
+## Suggested scrape configuration
+
+Configure your Prometheus-compatible scraper to use the application URL, HTTPS in production, and an authorization mechanism that securely supplies a short-lived admin token. Do not commit tokens or other credentials to prometheus.yml. If your scraper cannot safely rotate a Bearer token, place a trusted authentication proxy in front of the endpoint and restrict direct network access.
+
+## Initial alert guidance
+
+Treat these as starting points and tune them to the service's normal traffic:
+
+- HTTP 5xx ratio remains above 5% for 5 minutes.
+- Request p95 latency exceeds the service's agreed SLO for 5 minutes.
+- swiftchain_mongodb_connection_ready is 0.
+- swiftchain_redis_healthy is 0.
+- Any swiftchain_circuit_breaker_open sample is 1.
+- swiftchain_indexer_lag_ledgers remains above INDEXER_LAG_ALERT_THRESHOLD.
+- MongoDB pool in-use connections remain close to total connections while request latency rises.
+
+This endpoint exports metrics; it does not configure Prometheus, Grafana, Datadog, alert delivery, or retention automatically.

From 828f2673c68efbd75c65e633d0b8f7277f3baa72 Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:46:58 +0100
Subject: [PATCH 12/37] test(metrics): cover Prometheus exposition and
 collectors

---
 tests/unit/prometheusMetricsService.test.ts | 88 +++++++++++++++++++++
 1 file changed, 88 insertions(+)
 create mode 100644 tests/unit/prometheusMetricsService.test.ts

diff --git a/tests/unit/prometheusMetricsService.test.ts b/tests/unit/prometheusMetricsService.test.ts
new file mode 100644
index 0000000..c3f0399
--- /dev/null
+++ b/tests/unit/prometheusMetricsService.test.ts
@@ -0,0 +1,88 @@
+jest.mock('mongoose', () => ({
+  connection: {
+    readyState: 1,
+    getClient: jest.fn(() => ({ topology: { s: { servers: new Map() } } })),
+  },
+}));
+
+jest.mock('../../src/config/redis', () => ({
+  redisClient: {
+    status: 'ready',
+    ping: jest.fn().mockResolvedValue('PONG'),
+  },
+}));
+
+jest.mock('../../src/utils/circuitBreaker', () => ({
+  getAllCircuitBreakerStatuses: jest.fn(() => [
+    { name: 'soroban-rpc', state: 'open', stats: {} },
+    { name: 'database', state: 'closed', stats: {} },
+  ]),
+}));
+
+jest.mock('../../src/services/socketMetricsService', () => ({
+  __esModule: true,
+  default: {
+    getMetrics: jest.fn(() => ({
+      connectedSockets: 3,
+      totalConnections: 12,
+      totalDisconnections: 9,
+      messagesProcessed: 44,
+      messageLatencyMs: { p50: 5, p95: 15, p99: 20 },
+      memoryUsageBytes: { heapUsedMB: 10, heapTotalMB: 20, rssMB: 30, externalMB: 1 },
+      timestamp: '2026-01-01T00:00:00.000Z',
+    })),
+  },
+}));
+
+jest.mock('../../src/services/monitorService', () => ({
+  getIndexerLagSnapshot: jest.fn().mockResolvedValue({
+    network: 'testnet',
+    processedLedger: 100,
+    networkLedger: 125,
+    lagLedgers: 25,
+  }),
+}));
+
+import prometheusMetricsService from '../../src/services/prometheusMetricsService';
+
+describe('PrometheusMetricsService', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it('renders valid Prometheus declarations and request histogram samples', async () => {
+    prometheusMetricsService.recordRequest('get', '/api/v1/drivers/:driverId', 200, 30);
+
+    const output = await prometheusMetricsService.render();
+
+    expect(output).toContain('# TYPE swiftchain_http_requests_total counter');
+    expect(output).toContain('# TYPE swiftchain_http_request_duration_seconds histogram');
+    expect(output).toContain(
+      'swiftchain_http_requests_total{method="GET",route="/api/v1/drivers/:driverId",status_code="200"} 1',
+    );
+    expect(output).toContain(
+      'swiftchain_http_request_duration_seconds_bucket{method="GET",route="/api/v1/drivers/:driverId",le="0.05"} 1',
+    );
+    expect(output).toContain(
+      'swiftchain_http_request_duration_seconds_count{method="GET",route="/api/v1/drivers/:driverId"} 1',
+    );
+  });
+
+  it('includes operational dependency and existing Socket.IO metrics', async () => {
+    const output = await prometheusMetricsService.render();
+
+    expect(output).toContain('swiftchain_mongodb_connection_ready 1');
+    expect(output).toContain('swiftchain_redis_healthy 1');
+    expect(output).toContain('swiftchain_circuit_breaker_open{name="soroban-rpc",state="open"} 1');
+    expect(output).toContain('swiftchain_indexer_lag_ledgers 25');
+    expect(output).toContain('swiftchain_socket_connected_sockets 3');
+  });
+
+  it('escapes label values before rendering', async () => {
+    prometheusMetricsService.recordRequest('GET', '/api/v1/items/"special"', 500, 10);
+
+    const output = await prometheusMetricsService.render();
+
+    expect(output).toContain('route="/api/v1/items/\\\\"special\\\\""');
+  });
+});

From b86e611a2668f0fc8fd957945ca8b0dea682af36 Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:47:11 +0100
Subject: [PATCH 13/37] fix(metrics): handle unavailable MongoDB pool stats

---
 src/services/prometheusMetricsService.ts | 56 +++++++++++++-----------
 1 file changed, 30 insertions(+), 26 deletions(-)

diff --git a/src/services/prometheusMetricsService.ts b/src/services/prometheusMetricsService.ts
index ff67ff7..7fda25e 100644
--- a/src/services/prometheusMetricsService.ts
+++ b/src/services/prometheusMetricsService.ts
@@ -28,37 +28,41 @@ const safeMetricLabel = (value: string): string =>
   value.replace(/[^a-zA-Z0-9_.-]/g, '_').slice(0, 100) || 'unknown';
 
 const readMongoPoolSnapshot = (): MongoPoolSnapshot => {
-  const connection = mongoose.connection;
-  const topology = (connection.getClient() as unknown as {
-    topology?: {
-      s?: {
-        servers?: Map<string, {
-          s?: {
-            pool?: {
-              totalConnectionCount?: number;
-              availableConnectionCount?: number;
+  try {
+    const connection = mongoose.connection;
+    const topology = (connection.getClient() as unknown as {
+      topology?: {
+        s?: {
+          servers?: Map<string, {
+            s?: {
+              pool?: {
+                totalConnectionCount?: number;
+                availableConnectionCount?: number;
+              };
             };
-          };
-        }>;
+          }>;
+        };
       };
-    };
-  }).topology;
+    }).topology;
 
-  const servers = topology?.s?.servers;
-  if (!servers || typeof servers.values !== 'function') {
-    return { total: 0, available: 0, introspectionAvailable: false };
-  }
+    const servers = topology?.s?.servers;
+    if (!servers || typeof servers.values !== 'function') {
+      return { total: 0, available: 0, introspectionAvailable: false };
+    }
 
-  let total = 0;
-  let available = 0;
-  for (const server of servers.values()) {
-    const pool = server.s?.pool;
-    if (!pool) continue;
-    total += pool.totalConnectionCount ?? 0;
-    available += pool.availableConnectionCount ?? 0;
-  }
+    let total = 0;
+    let available = 0;
+    for (const server of servers.values()) {
+      const pool = server.s?.pool;
+      if (!pool) continue;
+      total += pool.totalConnectionCount ?? 0;
+      available += pool.availableConnectionCount ?? 0;
+    }
 
-  return { total, available, introspectionAvailable: true };
+    return { total, available, introspectionAvailable: true };
+  } catch {
+    return { total: 0, available: 0, introspectionAvailable: false };
+  }
 };
 
 /**

From 66a7f8d6cb80afb589fee1dbb1b4205d5e583532 Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:47:18 +0100
Subject: [PATCH 14/37] docs(metrics): link Prometheus operations guide from
 README

---
 README.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/README.md b/README.md
index a08e164..2f9198c 100644
--- a/README.md
+++ b/README.md
@@ -355,3 +355,8 @@ linked PR 5,4,6
 ## 📄 License
 
 This project is licensed under the ISC License.
+
+
+## Prometheus metrics endpoint
+
+Operational metrics are available at `GET /metrics` and `GET /api/v1/metrics` when `ENABLE_METRICS=true`. The endpoint is disabled by default and requires a valid admin Bearer token when enabled. See [Prometheus metrics and operations guidance](docs/prometheus-metrics.md) for metric names, scrape setup, and alert recommendations.

From 5381c123c22c5d49645bf26289a7830783bd1ffa Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:47:46 +0100
Subject: [PATCH 15/37] chore: remove placeholder PR draft from metrics change

---
 .commit_message.txt | 30 ------------------------------
 1 file changed, 30 deletions(-)
 delete mode 100644 .commit_message.txt

diff --git a/.commit_message.txt b/.commit_message.txt
deleted file mode 100644
index 27f0bef..0000000
--- a/.commit_message.txt
+++ /dev/null
@@ -1,30 +0,0 @@
-feat(#112): Implement Socket.IO load tests for 10,000 concurrent connections
-
-
-
-- Add metrics collection service (/api/v1/socket-metrics) with real-time latency tracking
-- Create k6 load test scenario with 10k VU ramp profile and latency assertions
-- Implement optimized data seeding scripts (default 25/25/50, 10k variants)
-- Integrate metrics recording in Socket.IO connection and location handlers
-- Add comprehensive README documentation with usage guides and troubleshooting
-- Include socketMetricsClient helper library for k6 metric assertions
-
-Architecture:
-- Controller -> Service pattern (socketMetricsController, socketMetricsService)
-- In-memory metrics with rolling latency window (no external dependencies)
-- Setup/teardown phases in k6 test for endpoint validation and output summary
-
-Deliverables:
-- socket-load.js: k6 scenario (stages: 0->2.5k->5k->10k VUs, soak 60s, drain 30s)
-- socketMetricsService: In-memory collection with p50/p95/p99 latency percentiles
-- seedLoadTestData10k.ts: Batch insertion with progress for large fixtures
-- Updated README with complete Socket.IO test documentation
-- SOCKET_IO_LOAD_TEST_PR.md: Comprehensive PR summary with deployment guidance
-
-Scripts:
-- npm run seed:10k: Seed 10k drivers, 5k deliveries, 100 customers
-- npm run test:socket: Run k6 load test
-- npm run test:socket:10k: Optimized for 10k concurrent test
-
-All code follows existing conventions: strict TypeScript, error handling,
-logging with context, production-quality implementation.

From 472045a33d5c8e463f9e6e7164a6a917f8967660 Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:47:53 +0100
Subject: [PATCH 16/37] test(metrics): correct escaped label assertion

---
 tests/unit/prometheusMetricsService.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unit/prometheusMetricsService.test.ts b/tests/unit/prometheusMetricsService.test.ts
index c3f0399..f1cebfe 100644
--- a/tests/unit/prometheusMetricsService.test.ts
+++ b/tests/unit/prometheusMetricsService.test.ts
@@ -83,6 +83,6 @@ describe('PrometheusMetricsService', () => {
 
     const output = await prometheusMetricsService.render();
 
-    expect(output).toContain('route="/api/v1/items/\\\\"special\\\\""');
+    expect(output).toContain('route="/api/v1/items/' + '\\' + '"special' + '\\' + '""');
   });
 });

From 8c0347201012208d4e6ab9ff31f3d222f3be74ea Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:48:32 +0100
Subject: [PATCH 17/37] fix(metrics): bound Redis probe and report pool
 visibility accurately

---
 src/services/prometheusMetricsService.ts | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/services/prometheusMetricsService.ts b/src/services/prometheusMetricsService.ts
index 7fda25e..315c09b 100644
--- a/src/services/prometheusMetricsService.ts
+++ b/src/services/prometheusMetricsService.ts
@@ -52,14 +52,16 @@ const readMongoPoolSnapshot = (): MongoPoolSnapshot => {
 
     let total = 0;
     let available = 0;
+    let poolsFound = 0;
     for (const server of servers.values()) {
       const pool = server.s?.pool;
       if (!pool) continue;
+      poolsFound += 1;
       total += pool.totalConnectionCount ?? 0;
       available += pool.availableConnectionCount ?? 0;
     }
 
-    return { total, available, introspectionAvailable: true };
+    return { total, available, introspectionAvailable: poolsFound > 0 };
   } catch {
     return { total: 0, available: 0, introspectionAvailable: false };
   }
@@ -173,10 +175,18 @@ class PrometheusMetricsService {
 
     let redisHealthy = 0;
     if (redisClient.status === 'ready') {
+      let timeout: ReturnType<typeof setTimeout> | undefined;
       try {
-        redisHealthy = (await redisClient.ping()) === 'PONG' ? 1 : 0;
+        const ping = redisClient.ping().then((response) => response === 'PONG');
+        const timedOut = new Promise<boolean>((resolve) => {
+          timeout = setTimeout(() => resolve(false), 1000);
+          timeout.unref?.();
+        });
+        redisHealthy = (await Promise.race([ping, timedOut])) ? 1 : 0;
       } catch {
         redisHealthy = 0;
+      } finally {
+        if (timeout) clearTimeout(timeout);
       }
     }
     gauge('swiftchain_redis_healthy', 'Whether Redis responds to a PING command.', redisHealthy);

From 43e1ebb2f91b0891e513907822a6e7e37293c916 Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:48:44 +0100
Subject: [PATCH 18/37] test(metrics): verify route labels avoid request
 identifiers

---
 .../unit/prometheusMetricsMiddleware.test.ts  | 58 +++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 tests/unit/prometheusMetricsMiddleware.test.ts

diff --git a/tests/unit/prometheusMetricsMiddleware.test.ts b/tests/unit/prometheusMetricsMiddleware.test.ts
new file mode 100644
index 0000000..30905bb
--- /dev/null
+++ b/tests/unit/prometheusMetricsMiddleware.test.ts
@@ -0,0 +1,58 @@
+import { EventEmitter } from 'events';
+import type { Request, Response } from 'express';
+import prometheusMetricsService from '../../src/services/prometheusMetricsService';
+import prometheusMetricsMiddleware from '../../src/middleware/prometheusMetricsMiddleware';
+
+jest.mock('../../src/services/prometheusMetricsService', () => ({
+  __esModule: true,
+  default: {
+    recordRequest: jest.fn(),
+  },
+}));
+
+describe('prometheusMetricsMiddleware', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it('records route templates instead of concrete request URLs', () => {
+    const req = {
+      path: '/api/v1/drivers/driver-123',
+      method: 'GET',
+      baseUrl: '/api/v1/drivers',
+      route: { path: '/:driverId' },
+    } as unknown as Request;
+    const res = new EventEmitter() as unknown as Response;
+    Object.defineProperty(res, 'statusCode', { value: 200, writable: true });
+    const next = jest.fn();
+
+    prometheusMetricsMiddleware(req, res, next);
+    res.emit('finish');
+
+    expect(next).toHaveBeenCalledTimes(1);
+    expect(prometheusMetricsService.recordRequest).toHaveBeenCalledWith(
+      'GET',
+      '/api/v1/drivers/:driverId',
+      200,
+      expect.any(Number),
+    );
+  });
+
+  it('does not record metrics scrapes as ordinary HTTP traffic', () => {
+    const req = {
+      path: '/metrics',
+      method: 'GET',
+      baseUrl: '',
+      route: { path: '/' },
+    } as unknown as Request;
+    const res = new EventEmitter() as unknown as Response;
+    Object.defineProperty(res, 'statusCode', { value: 200, writable: true });
+    const next = jest.fn();
+
+    prometheusMetricsMiddleware(req, res, next);
+    res.emit('finish');
+
+    expect(next).toHaveBeenCalledTimes(1);
+    expect(prometheusMetricsService.recordRequest).not.toHaveBeenCalled();
+  });
+});

From b38a37402be769e8bb794a56aa9e726c586c23fa Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:49:22 +0100
Subject: [PATCH 19/37] docs(metrics): add deployment guidance for protected
 metrics endpoint

---
 docs/deployment.md | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 docs/deployment.md

diff --git a/docs/deployment.md b/docs/deployment.md
new file mode 100644
index 0000000..a29e943
--- /dev/null
+++ b/docs/deployment.md
@@ -0,0 +1,23 @@
+# Deployment notes
+
+## Prometheus metrics endpoint
+
+The operational metrics endpoint is disabled by default. To enable it in a deployment environment, set the validated variable in the service environment:
+
+    ENABLE_METRICS=true
+
+Restart the application after changing this setting. No database migration is required.
+
+When enabled, the protected endpoint is available at GET /metrics and GET /api/v1/metrics. Both require a valid Bearer JWT for a user with the admin role. Use HTTPS and restrict the endpoint to monitoring infrastructure through a private network, firewall, or trusted authentication proxy. Do not place long-lived admin credentials in source control or commit scrape tokens to a Prometheus configuration file.
+
+A Prometheus-compatible scraper must send an Authorization: Bearer header. Use a secret file or a trusted proxy that can securely rotate credentials. If your deployment cannot supply rotating Bearer tokens safely, keep ENABLE_METRICS=false until an appropriate authentication boundary is configured.
+
+For exported metric names, scrape configuration guidance, and suggested alerts, see [Prometheus metrics](prometheus-metrics.md).
+
+## Operational safety
+
+- Confirm the endpoint returns 404 while ENABLE_METRICS is false.
+- Confirm a request without a valid admin token receives an authentication error when metrics are enabled.
+- Confirm an authenticated non-admin receives 403.
+- Confirm a valid admin token receives Prometheus text exposition with a text/plain content type.
+- Monitor scrape latency: the indexer lag collector reads the persisted checkpoint and requests the current network ledger, so scrapes may be slower when the Stellar RPC service is degraded.

From dfcbc6311ef3cf743592076e1bca1da5cbab5b94 Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:49:50 +0100
Subject: [PATCH 20/37] style(metrics): format metrics environment schema

---
 src/config/env.ts | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/config/env.ts b/src/config/env.ts
index ec6fe8a..20b027f 100644
--- a/src/config/env.ts
+++ b/src/config/env.ts
@@ -230,7 +230,12 @@ const envSchema = z.object({
   CORS_ORIGIN: z.string().default('*'),
   RATE_LIMIT_WINDOW_MS: numeric(z.coerce.number().int().min(1000).default(900000)),
   RATE_LIMIT_MAX_REQUESTS: numeric(z.coerce.number().int().min(1).default(100)),
-  ENABLE_METRICS: z.preprocess(blankToUndefined, z.enum(['true', 'false']).default('false').transform((value) => value === 'true')),
+  ENABLE_METRICS: z.preprocess(
+    blankToUndefined,
+    z.enum(['true', 'false'])
+      .default('false')
+      .transform((value) => value === 'true'),
+  ),
   DISPUTE_NOTIFICATION_WEBHOOK_URL: z.string().default(''),
 
   // ── Uploads / storage ──────────────────────────────────────────────────────

From 8c334d87da9986dbc4f3f752db6ad056fefc0fcf Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:49:53 +0100
Subject: [PATCH 21/37] style(metrics): format Prometheus collector output

---
 src/services/prometheusMetricsService.ts | 53 ++++++++++++++++++++----
 1 file changed, 44 insertions(+), 9 deletions(-)

diff --git a/src/services/prometheusMetricsService.ts b/src/services/prometheusMetricsService.ts
index 315c09b..593a8dd 100644
--- a/src/services/prometheusMetricsService.ts
+++ b/src/services/prometheusMetricsService.ts
@@ -22,7 +22,11 @@ const escapeLabel = (value: string): string =>
   value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n');
 
 const labelsToString = (labels: Record<string, string | number>): string =>
-  '{' + Object.entries(labels).map(([key, value]) => key + '="' + escapeLabel(String(value)) + '"').join(',') + '}';
+  '{' +
+  Object.entries(labels)
+    .map(([key, value]) => key + '="' + escapeLabel(String(value)) + '"')
+    .join(',') +
+  '}';
 
 const safeMetricLabel = (value: string): string =>
   value.replace(/[^a-zA-Z0-9_.-]/g, '_').slice(0, 100) || 'unknown';
@@ -168,7 +172,10 @@ class PrometheusMetricsService {
       '# HELP swiftchain_mongodb_pool_connections MongoDB pool connections by state.',
       '# TYPE swiftchain_mongodb_pool_connections gauge',
       'swiftchain_mongodb_pool_connections' + labelsToString({ state: 'total' }) + ' ' + mongoPool.total,
-      'swiftchain_mongodb_pool_connections' + labelsToString({ state: 'available' }) + ' ' + mongoPool.available,
+      'swiftchain_mongodb_pool_connections' +
+        labelsToString({ state: 'available' }) +
+        ' ' +
+        mongoPool.available,
       'swiftchain_mongodb_pool_connections' +
         labelsToString({ state: 'in_use' }) + ' ' + Math.max(0, mongoPool.total - mongoPool.available),
     );
@@ -219,13 +226,41 @@ class PrometheusMetricsService {
     }
 
     const socketMetrics = socketMetricsService.getMetrics();
-    gauge('swiftchain_socket_connected_sockets', 'Currently connected Socket.IO clients.', socketMetrics.connectedSockets);
-    gauge('swiftchain_socket_total_connections', 'Total Socket.IO connections since process start.', socketMetrics.totalConnections);
-    gauge('swiftchain_socket_total_disconnections', 'Total Socket.IO disconnections since process start.', socketMetrics.totalDisconnections);
-    gauge('swiftchain_socket_messages_processed', 'Total Socket.IO messages processed since process start.', socketMetrics.messagesProcessed);
-    gauge('swiftchain_socket_message_latency_p50_ms', 'Socket.IO message latency p50 in milliseconds.', socketMetrics.messageLatencyMs.p50);
-    gauge('swiftchain_socket_message_latency_p95_ms', 'Socket.IO message latency p95 in milliseconds.', socketMetrics.messageLatencyMs.p95);
-    gauge('swiftchain_socket_message_latency_p99_ms', 'Socket.IO message latency p99 in milliseconds.', socketMetrics.messageLatencyMs.p99);
+    gauge(
+      'swiftchain_socket_connected_sockets',
+      'Currently connected Socket.IO clients.',
+      socketMetrics.connectedSockets,
+    );
+    gauge(
+      'swiftchain_socket_total_connections',
+      'Total Socket.IO connections since process start.',
+      socketMetrics.totalConnections,
+    );
+    gauge(
+      'swiftchain_socket_total_disconnections',
+      'Total Socket.IO disconnections since process start.',
+      socketMetrics.totalDisconnections,
+    );
+    gauge(
+      'swiftchain_socket_messages_processed',
+      'Total Socket.IO messages processed since process start.',
+      socketMetrics.messagesProcessed,
+    );
+    gauge(
+      'swiftchain_socket_message_latency_p50_ms',
+      'Socket.IO message latency p50 in milliseconds.',
+      socketMetrics.messageLatencyMs.p50,
+    );
+    gauge(
+      'swiftchain_socket_message_latency_p95_ms',
+      'Socket.IO message latency p95 in milliseconds.',
+      socketMetrics.messageLatencyMs.p95,
+    );
+    gauge(
+      'swiftchain_socket_message_latency_p99_ms',
+      'Socket.IO message latency p99 in milliseconds.',
+      socketMetrics.messageLatencyMs.p99,
+    );
 
     return lines.join('\n') + '\n';
   }

From 1cf795b677304d72b3212f4af2a82720a9ba51c6 Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:50:00 +0100
Subject: [PATCH 22/37] style(metrics): wrap long Prometheus assertion

---
 tests/unit/prometheusMetricsService.test.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/unit/prometheusMetricsService.test.ts b/tests/unit/prometheusMetricsService.test.ts
index f1cebfe..aabb82c 100644
--- a/tests/unit/prometheusMetricsService.test.ts
+++ b/tests/unit/prometheusMetricsService.test.ts
@@ -61,7 +61,8 @@ describe('PrometheusMetricsService', () => {
       'swiftchain_http_requests_total{method="GET",route="/api/v1/drivers/:driverId",status_code="200"} 1',
     );
     expect(output).toContain(
-      'swiftchain_http_request_duration_seconds_bucket{method="GET",route="/api/v1/drivers/:driverId",le="0.05"} 1',
+      'swiftchain_http_request_duration_seconds_bucket' +
+        '{method="GET",route="/api/v1/drivers/:driverId",le="0.05"} 1',
     );
     expect(output).toContain(
       'swiftchain_http_request_duration_seconds_count{method="GET",route="/api/v1/drivers/:driverId"} 1',

From f832986825e76365bbfc18fcef3b53435bb5d8fb Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:50:19 +0100
Subject: [PATCH 23/37] chore: preserve repository commit message template

---
 .commit_message.txt | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 .commit_message.txt

diff --git a/.commit_message.txt b/.commit_message.txt
new file mode 100644
index 0000000..cd3b5d9
--- /dev/null
+++ b/.commit_message.txt
@@ -0,0 +1,28 @@
+feat(#112): Implement Socket.IO load tests for 10,000 concurrent connections
+
+- Add metrics collection service (/api/v1/socket-metrics) with real-time latency tracking
+- Create k6 load test scenario with 10k VU ramp profile and latency assertions
+- Implement optimized data seeding scripts (default 25/25/50, 10k variants)
+- Integrate metrics recording in Socket.IO connection and location handlers
+- Add comprehensive README documentation with usage guides and troubleshooting
+- Include socketMetricsClient helper library for k6 metric assertions
+
+Architecture:
+- Controller -> Service pattern (socketMetricsController, socketMetricsService)
+- In-memory metrics with rolling latency window (no external dependencies)
+- Setup/teardown phases in k6 test for endpoint validation and output summary
+
+Deliverables:
+- socket-load.js: k6 scenario (stages: 0->2.5k->5k->10k VUs, soak 60s, drain 30s)
+- socketMetricsService: In-memory collection with p50/p95/p99 latency percentiles
+- seedLoadTestData10k.ts: Batch insertion with progress for large fixtures
+- Updated README with complete Socket.IO test documentation
+- SOCKET_IO_LOAD_TEST_PR.md: Comprehensive PR summary with deployment guidance
+
+Scripts:
+- npm run seed:10k: Seed 10k drivers, 5k deliveries, 100 customers
+- npm run test:socket: Run k6 load test
+- npm run test:socket:10k: Optimized for 10k concurrent test
+
+All code follows existing conventions: strict TypeScript, error handling,
+logging with context, production-quality implementation.

From a1079951dfcc9143587db97e27db5d3d4dd17255 Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 11:08:02 +0100
Subject: [PATCH 24/37] fix(metrics): declare cumulative Socket.IO metrics as
 counters

---
 src/services/prometheusMetricsService.ts | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/services/prometheusMetricsService.ts b/src/services/prometheusMetricsService.ts
index 593a8dd..a50e66c 100644
--- a/src/services/prometheusMetricsService.ts
+++ b/src/services/prometheusMetricsService.ts
@@ -114,6 +114,13 @@ class PrometheusMetricsService {
         name + ' ' + (Number.isFinite(value) ? value : 0),
       );
     };
+    const counter = (name: string, help: string, value: number): void => {
+      lines.push(
+        '# HELP ' + name + ' ' + help,
+        '# TYPE ' + name + ' counter',
+        name + ' ' + (Number.isFinite(value) ? value : 0),
+      );
+    };
 
     lines.push(
       '# HELP swiftchain_http_requests_total Total HTTP requests served by route.',
@@ -231,17 +238,17 @@ class PrometheusMetricsService {
       'Currently connected Socket.IO clients.',
       socketMetrics.connectedSockets,
     );
-    gauge(
+    counter(
       'swiftchain_socket_total_connections',
       'Total Socket.IO connections since process start.',
       socketMetrics.totalConnections,
     );
-    gauge(
+    counter(
       'swiftchain_socket_total_disconnections',
       'Total Socket.IO disconnections since process start.',
       socketMetrics.totalDisconnections,
     );
-    gauge(
+    counter(
       'swiftchain_socket_messages_processed',
       'Total Socket.IO messages processed since process start.',
       socketMetrics.messagesProcessed,

From 313569a46447fd61afc562c87f24ec2184d4b084 Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 11:08:31 +0100
Subject: [PATCH 25/37] fix(metrics): exclude metrics route variants from
 request counters

---
 src/middleware/prometheusMetricsMiddleware.ts | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/middleware/prometheusMetricsMiddleware.ts b/src/middleware/prometheusMetricsMiddleware.ts
index 09aa10a..a5c07c2 100644
--- a/src/middleware/prometheusMetricsMiddleware.ts
+++ b/src/middleware/prometheusMetricsMiddleware.ts
@@ -11,7 +11,12 @@ const prometheusMetricsMiddleware = (
   next: NextFunction,
 ): void => {
   const path = req.path;
-  if (path === '/metrics' || path === '/api/v1/metrics' || path === '/api/v1/metrics/') {
+  if (
+    path === '/metrics' ||
+    path === '/metrics/' ||
+    path === '/api/v1/metrics' ||
+    path === '/api/v1/metrics/'
+  ) {
     next();
     return;
   }

From 303897dfbd18df3512e8d342b329f2f1b91b4d69 Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 11:08:39 +0100
Subject: [PATCH 26/37] test(metrics): assert cumulative socket metrics use
 counter type

---
 tests/unit/prometheusMetricsService.test.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/unit/prometheusMetricsService.test.ts b/tests/unit/prometheusMetricsService.test.ts
index aabb82c..c6917cf 100644
--- a/tests/unit/prometheusMetricsService.test.ts
+++ b/tests/unit/prometheusMetricsService.test.ts
@@ -77,6 +77,7 @@ describe('PrometheusMetricsService', () => {
     expect(output).toContain('swiftchain_circuit_breaker_open{name="soroban-rpc",state="open"} 1');
     expect(output).toContain('swiftchain_indexer_lag_ledgers 25');
     expect(output).toContain('swiftchain_socket_connected_sockets 3');
+    expect(output).toContain('# TYPE swiftchain_socket_total_connections counter');
   });
 
   it('escapes label values before rendering', async () => {

From b2580ed6d530e553a07079162ac60e0e100e477d Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 11:08:58 +0100
Subject: [PATCH 27/37] fix(metrics): handle degraded RPC responses in lag
 snapshot

---
 src/services/monitorService.ts | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/services/monitorService.ts b/src/services/monitorService.ts
index ca34728..de80595 100644
--- a/src/services/monitorService.ts
+++ b/src/services/monitorService.ts
@@ -180,7 +180,10 @@ export const getIndexerLagSnapshot = async (): Promise<IndexerLagSnapshot | null
     return null;
   }
 
-  const networkLedger = await sorobanService.getLatestLedger();
+  const rawNetworkLedger = await sorobanService.getLatestLedger();
+  const networkLedger =
+    typeof rawNetworkLedger === 'number' ? rawNetworkLedger : status.lastProcessedLedger;
+
   return {
     network: status.network,
     processedLedger: status.lastProcessedLedger,

From 7a3e5161ec3a79343c5c6e0f22ffc0324cd5c9ee Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 11:10:03 +0100
Subject: [PATCH 28/37] fix(metrics): mark degraded RPC lag snapshots
 unavailable

---
 src/services/monitorService.ts | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/services/monitorService.ts b/src/services/monitorService.ts
index de80595..84d87af 100644
--- a/src/services/monitorService.ts
+++ b/src/services/monitorService.ts
@@ -181,14 +181,15 @@ export const getIndexerLagSnapshot = async (): Promise<IndexerLagSnapshot | null
   }
 
   const rawNetworkLedger = await sorobanService.getLatestLedger();
-  const networkLedger =
-    typeof rawNetworkLedger === 'number' ? rawNetworkLedger : status.lastProcessedLedger;
+  if (typeof rawNetworkLedger !== 'number') {
+    return null;
+  }
 
   return {
     network: status.network,
     processedLedger: status.lastProcessedLedger,
-    networkLedger,
-    lagLedgers: Math.max(0, networkLedger - status.lastProcessedLedger),
+    networkLedger: rawNetworkLedger,
+    lagLedgers: Math.max(0, rawNetworkLedger - status.lastProcessedLedger),
   };
 };
 

From c15d7dc10b8a8f784fcce68015a59b01b870beec Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 11:15:07 +0100
Subject: [PATCH 29/37] style(metrics): format Prometheus metrics service

---
 src/services/prometheusMetricsService.ts | 90 ++++++++++++++++++------
 1 file changed, 68 insertions(+), 22 deletions(-)

diff --git a/src/services/prometheusMetricsService.ts b/src/services/prometheusMetricsService.ts
index a50e66c..0b7f93b 100644
--- a/src/services/prometheusMetricsService.ts
+++ b/src/services/prometheusMetricsService.ts
@@ -37,14 +37,17 @@ const readMongoPoolSnapshot = (): MongoPoolSnapshot => {
     const topology = (connection.getClient() as unknown as {
       topology?: {
         s?: {
-          servers?: Map<string, {
-            s?: {
-              pool?: {
-                totalConnectionCount?: number;
-                availableConnectionCount?: number;
+          servers?: Map<
+            string,
+            {
+              s?: {
+                pool?: {
+                  totalConnectionCount?: number;
+                  availableConnectionCount?: number;
+                };
               };
-            };
-          }>;
+            }
+          >;
         };
       };
     }).topology;
@@ -80,7 +83,12 @@ class PrometheusMetricsService {
   private readonly requestCounters = new Map<string, number>();
   private readonly requestHistograms = new Map<string, HistogramSnapshot>();
 
-  public recordRequest(method: string, route: string, statusCode: number, durationMs: number): void {
+  public recordRequest(
+    method: string,
+    route: string,
+    statusCode: number,
+    durationMs: number,
+  ): void {
     const normalizedMethod = method.toUpperCase();
     const normalizedRoute = route || 'unmatched';
     const counterKey = JSON.stringify([normalizedMethod, normalizedRoute, statusCode]);
@@ -131,7 +139,8 @@ class PrometheusMetricsService {
       lines.push(
         'swiftchain_http_requests_total' +
           labelsToString({ method, route, status_code: statusCode }) +
-          ' ' + value,
+          ' ' +
+          value,
       );
     }
 
@@ -145,24 +154,47 @@ class PrometheusMetricsService {
         lines.push(
           'swiftchain_http_request_duration_seconds_bucket' +
             labelsToString({ method, route, le: boundary }) +
-            ' ' + histogram.bucketCounts[index],
+            ' ' +
+            histogram.bucketCounts[index],
         );
       });
       lines.push(
         'swiftchain_http_request_duration_seconds_bucket' +
-          labelsToString({ method, route, le: '+Inf' }) + ' ' + histogram.count,
+          labelsToString({ method, route, le: '+Inf' }) +
+          ' ' +
+          histogram.count,
         'swiftchain_http_request_duration_seconds_sum' +
-          labelsToString({ method, route }) + ' ' + histogram.sumSeconds,
+          labelsToString({ method, route }) +
+          ' ' +
+          histogram.sumSeconds,
         'swiftchain_http_request_duration_seconds_count' +
-          labelsToString({ method, route }) + ' ' + histogram.count,
+          labelsToString({ method, route }) +
+          ' ' +
+          histogram.count,
       );
     }
 
-    gauge('swiftchain_process_uptime_seconds', 'Process uptime in seconds.', process.uptime());
+    gauge(
+      'swiftchain_process_uptime_seconds',
+      'Process uptime in seconds.',
+      process.uptime(),
+    );
     const memory = process.memoryUsage();
-    gauge('swiftchain_process_resident_memory_bytes', 'Resident process memory in bytes.', memory.rss);
-    gauge('swiftchain_process_heap_used_bytes', 'Node.js heap memory used in bytes.', memory.heapUsed);
-    gauge('swiftchain_process_heap_total_bytes', 'Node.js heap memory allocated in bytes.', memory.heapTotal);
+    gauge(
+      'swiftchain_process_resident_memory_bytes',
+      'Resident process memory in bytes.',
+      memory.rss,
+    );
+    gauge(
+      'swiftchain_process_heap_used_bytes',
+      'Node.js heap memory used in bytes.',
+      memory.heapUsed,
+    );
+    gauge(
+      'swiftchain_process_heap_total_bytes',
+      'Node.js heap memory allocated in bytes.',
+      memory.heapTotal,
+    );
 
     const mongoPool = readMongoPoolSnapshot();
     gauge(
@@ -178,13 +210,18 @@ class PrometheusMetricsService {
     lines.push(
       '# HELP swiftchain_mongodb_pool_connections MongoDB pool connections by state.',
       '# TYPE swiftchain_mongodb_pool_connections gauge',
-      'swiftchain_mongodb_pool_connections' + labelsToString({ state: 'total' }) + ' ' + mongoPool.total,
+      'swiftchain_mongodb_pool_connections' +
+        labelsToString({ state: 'total' }) +
+        ' ' +
+        mongoPool.total,
       'swiftchain_mongodb_pool_connections' +
         labelsToString({ state: 'available' }) +
         ' ' +
         mongoPool.available,
       'swiftchain_mongodb_pool_connections' +
-        labelsToString({ state: 'in_use' }) + ' ' + Math.max(0, mongoPool.total - mongoPool.available),
+        labelsToString({ state: 'in_use' }) +
+        ' ' +
+        Math.max(0, mongoPool.total - mongoPool.available),
     );
 
     let redisHealthy = 0;
@@ -214,13 +251,18 @@ class PrometheusMetricsService {
       lines.push(
         'swiftchain_circuit_breaker_open' +
           labelsToString({ name: safeMetricLabel(breaker.name), state: breaker.state }) +
-          ' ' + (breaker.state === 'closed' ? 0 : 1),
+          ' ' +
+          (breaker.state === 'closed' ? 0 : 1),
       );
     }
 
     try {
       const lag = await getIndexerLagSnapshot();
-      gauge('swiftchain_indexer_lag_available', 'Whether indexer lag could be measured.', lag ? 1 : 0);
+      gauge(
+        'swiftchain_indexer_lag_available',
+        'Whether indexer lag could be measured.',
+        lag ? 1 : 0,
+      );
       if (lag) {
         gauge(
           'swiftchain_indexer_lag_ledgers',
@@ -229,7 +271,11 @@ class PrometheusMetricsService {
         );
       }
     } catch {
-      gauge('swiftchain_indexer_lag_available', 'Whether indexer lag could be measured.', 0);
+      gauge(
+        'swiftchain_indexer_lag_available',
+        'Whether indexer lag could be measured.',
+        0,
+      );
     }
 
     const socketMetrics = socketMetricsService.getMetrics();

From 4e1427b0dd905a1e59d0cd4c29ccf051e6b5db30 Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 11:15:09 +0100
Subject: [PATCH 30/37] style(metrics): format request metrics middleware

---
 src/middleware/prometheusMetricsMiddleware.ts | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/src/middleware/prometheusMetricsMiddleware.ts b/src/middleware/prometheusMetricsMiddleware.ts
index a5c07c2..261d119 100644
--- a/src/middleware/prometheusMetricsMiddleware.ts
+++ b/src/middleware/prometheusMetricsMiddleware.ts
@@ -5,11 +5,7 @@ import prometheusMetricsService from '../services/prometheusMetricsService';
  * Records completed HTTP requests using route templates to avoid unbounded
  * cardinality from identifiers and query parameters.
  */
-const prometheusMetricsMiddleware = (
-  req: Request,
-  res: Response,
-  next: NextFunction,
-): void => {
+const prometheusMetricsMiddleware = (req: Request, res: Response, next: NextFunction): void => {
   const path = req.path;
   if (
     path === '/metrics' ||
@@ -24,9 +20,10 @@ const prometheusMetricsMiddleware = (
   const startedAt = process.hrtime.bigint();
   res.once('finish', () => {
     const durationMs = Number(process.hrtime.bigint() - startedAt) / 1_000_000;
-    const routePath = req.route && typeof req.route.path === 'string'
-      ? (req.baseUrl + req.route.path) || req.route.path
-      : 'unmatched';
+    const routePath =
+      req.route && typeof req.route.path === 'string'
+        ? req.baseUrl + req.route.path || req.route.path
+        : 'unmatched';
     prometheusMetricsService.recordRequest(req.method, routePath, res.statusCode, durationMs);
   });
 

From b422d6b42fd70d50e992c2f234712315aa128ef0 Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 11:15:12 +0100
Subject: [PATCH 31/37] style(config): format indexer lag threshold

---
 src/config/env.ts | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/config/env.ts b/src/config/env.ts
index 20b027f..1b37024 100644
--- a/src/config/env.ts
+++ b/src/config/env.ts
@@ -272,7 +272,11 @@ const envSchema = z.object({
   AWS_S3_SIGNED_URL_EXPIRES_SECONDS: z.coerce.number().int().min(1).default(900),
 
   // ── Indexer lag monitoring ─────────────────────────────────────────────────
-  INDEXER_LAG_ALERT_THRESHOLD: z.coerce.number().int().min(1).default(100),
+  INDEXER_LAG_ALERT_THRESHOLD: z.coerce
+    .number()
+    .int()
+    .min(1)
+    .default(100),
   INDEXER_LAG_CHECK_INTERVAL_MS: z.coerce.number().int().min(1000).default(60000),
   INDEXER_LAG_WEBHOOK_URL: z.string().trim().default(''),
 

From d4c57b2235a6d5347e789196a6fa397eef03cadc Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 11:17:42 +0100
Subject: [PATCH 32/37] style(metrics): use Prometheus counter naming
 convention

---
 src/services/prometheusMetricsService.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/services/prometheusMetricsService.ts b/src/services/prometheusMetricsService.ts
index 0b7f93b..c927c29 100644
--- a/src/services/prometheusMetricsService.ts
+++ b/src/services/prometheusMetricsService.ts
@@ -285,17 +285,17 @@ class PrometheusMetricsService {
       socketMetrics.connectedSockets,
     );
     counter(
-      'swiftchain_socket_total_connections',
+      'swiftchain_socket_connections_total',
       'Total Socket.IO connections since process start.',
       socketMetrics.totalConnections,
     );
     counter(
-      'swiftchain_socket_total_disconnections',
+      'swiftchain_socket_disconnections_total',
       'Total Socket.IO disconnections since process start.',
       socketMetrics.totalDisconnections,
     );
     counter(
-      'swiftchain_socket_messages_processed',
+      'swiftchain_socket_messages_processed_total',
       'Total Socket.IO messages processed since process start.',
       socketMetrics.messagesProcessed,
     );

From aa706b2425def63d7143779a6a8c0a1ddacba3e8 Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 11:17:51 +0100
Subject: [PATCH 33/37] test(metrics): align counter assertions with metric
 names

---
 tests/unit/prometheusMetricsService.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unit/prometheusMetricsService.test.ts b/tests/unit/prometheusMetricsService.test.ts
index c6917cf..d9955d5 100644
--- a/tests/unit/prometheusMetricsService.test.ts
+++ b/tests/unit/prometheusMetricsService.test.ts
@@ -77,7 +77,7 @@ describe('PrometheusMetricsService', () => {
     expect(output).toContain('swiftchain_circuit_breaker_open{name="soroban-rpc",state="open"} 1');
     expect(output).toContain('swiftchain_indexer_lag_ledgers 25');
     expect(output).toContain('swiftchain_socket_connected_sockets 3');
-    expect(output).toContain('# TYPE swiftchain_socket_total_connections counter');
+    expect(output).toContain('# TYPE swiftchain_socket_connections_total counter');
   });
 
   it('escapes label values before rendering', async () => {

From 345785e12817237603297756dea46371d529541d Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 11:18:01 +0100
Subject: [PATCH 34/37] docs(metrics): clarify Socket.IO metric types

---
 docs/prometheus-metrics.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/prometheus-metrics.md b/docs/prometheus-metrics.md
index 0582bf0..6fdf362 100644
--- a/docs/prometheus-metrics.md
+++ b/docs/prometheus-metrics.md
@@ -37,7 +37,7 @@ The response uses text/plain; version=0.0.4; charset=utf-8 and can be scraped by
 | swiftchain_circuit_breaker_open | Gauge | Circuit-breaker state by registered breaker |
 | swiftchain_indexer_lag_available | Gauge | Whether a lag snapshot could be read |
 | swiftchain_indexer_lag_ledgers | Gauge | Gap between persisted processed ledger and the live network ledger |
-| swiftchain_socket_* | Gauge | Existing Socket.IO connection, message, and latency snapshots |
+| swiftchain_socket_* | Gauge / counter | Current connection and latency snapshots are gauges; cumulative connection, disconnection, and message totals are counters |
 
 HTTP route labels use Express route templates rather than raw request URLs, preventing user IDs, arbitrary paths, and query values from creating high-cardinality labels. The indexer metric reads the current checkpoint and live ledger without persisting alerts or sending webhooks during a scrape.
 

From 43555dd899a6a19fc993d976e05dff5b08c648b7 Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 11:18:20 +0100
Subject: [PATCH 35/37] docs(metrics): clarify per-process metric lifecycle

---
 docs/prometheus-metrics.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/prometheus-metrics.md b/docs/prometheus-metrics.md
index 6fdf362..7a96300 100644
--- a/docs/prometheus-metrics.md
+++ b/docs/prometheus-metrics.md
@@ -59,4 +59,4 @@ Treat these as starting points and tune them to the service's normal traffic:
 - swiftchain_indexer_lag_ledgers remains above INDEXER_LAG_ALERT_THRESHOLD.
 - MongoDB pool in-use connections remain close to total connections while request latency rises.
 
-This endpoint exports metrics; it does not configure Prometheus, Grafana, Datadog, alert delivery, or retention automatically.
+All in-memory request counters and socket totals are process-local and reset when that instance restarts. Scrape each instance separately and aggregate in the monitoring backend; do not treat one instance's values as a fleet-wide total. This endpoint exports metrics; it does not configure Prometheus, Grafana, Datadog, alert delivery, or retention automatically.

From 3942b312c39407c04f6bc8b41c8e8e6bfed92bdc Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 11:22:17 +0100
Subject: [PATCH 36/37] test(metrics): cover dependency failure and pool
 telemetry paths

---
 tests/unit/prometheusMetricsService.test.ts | 63 +++++++++++++++++++++
 1 file changed, 63 insertions(+)

diff --git a/tests/unit/prometheusMetricsService.test.ts b/tests/unit/prometheusMetricsService.test.ts
index d9955d5..33c5f91 100644
--- a/tests/unit/prometheusMetricsService.test.ts
+++ b/tests/unit/prometheusMetricsService.test.ts
@@ -44,10 +44,28 @@ jest.mock('../../src/services/monitorService', () => ({
 }));
 
 import prometheusMetricsService from '../../src/services/prometheusMetricsService';
+import mongoose from 'mongoose';
+import { redisClient } from '../../src/config/redis';
+import { getIndexerLagSnapshot } from '../../src/services/monitorService';
+
+const mongoConnection = (mongoose as unknown as { connection: { getClient: jest.Mock } }).connection;
+const redis = redisClient as unknown as { status: string; ping: jest.Mock };
+const indexerLagSnapshot = getIndexerLagSnapshot as jest.Mock;
 
 describe('PrometheusMetricsService', () => {
   beforeEach(() => {
     jest.clearAllMocks();
+    mongoConnection.getClient.mockImplementation(() => ({
+      topology: { s: { servers: new Map() } },
+    }));
+    redis.status = 'ready';
+    redis.ping.mockResolvedValue('PONG');
+    indexerLagSnapshot.mockResolvedValue({
+      network: 'testnet',
+      processedLedger: 100,
+      networkLedger: 125,
+      lagLedgers: 25,
+    });
   });
 
   it('renders valid Prometheus declarations and request histogram samples', async () => {
@@ -80,6 +98,51 @@ describe('PrometheusMetricsService', () => {
     expect(output).toContain('# TYPE swiftchain_socket_connections_total counter');
   });
 
+  it('exports MongoDB pool utilization when driver pool statistics are available', async () => {
+    mongoConnection.getClient.mockReturnValue({
+      topology: {
+        s: {
+          servers: new Map([
+            ['primary', { s: { pool: { totalConnectionCount: 10, availableConnectionCount: 4 } } }],
+          ]),
+        },
+      },
+    });
+
+    const output = await prometheusMetricsService.render();
+
+    expect(output).toContain('swiftchain_mongodb_pool_stats_available 1');
+    expect(output).toContain('swiftchain_mongodb_pool_connections{state="total"} 10');
+    expect(output).toContain('swiftchain_mongodb_pool_connections{state="available"} 4');
+    expect(output).toContain('swiftchain_mongodb_pool_connections{state="in_use"} 6');
+  });
+
+  it('reports unavailable dependencies without failing the metrics scrape', async () => {
+    mongoConnection.getClient.mockImplementation(() => {
+      throw new Error('MongoDB is disconnected');
+    });
+    redis.status = 'wait';
+    indexerLagSnapshot.mockResolvedValueOnce(null);
+
+    const output = await prometheusMetricsService.render();
+
+    expect(output).toContain('swiftchain_mongodb_pool_stats_available 0');
+    expect(output).toContain('swiftchain_redis_healthy 0');
+    expect(output).toContain('swiftchain_indexer_lag_available 0');
+    expect(output).not.toContain('swiftchain_indexer_lag_ledgers ');
+  });
+
+  it('handles Redis and indexer probe errors without breaking exposition output', async () => {
+    redis.ping.mockRejectedValueOnce(new Error('Redis unavailable'));
+    indexerLagSnapshot.mockRejectedValueOnce(new Error('RPC unavailable'));
+
+    const output = await prometheusMetricsService.render();
+
+    expect(output).toContain('swiftchain_redis_healthy 0');
+    expect(output).toContain('swiftchain_indexer_lag_available 0');
+    expect(output).toContain('# TYPE swiftchain_http_requests_total counter');
+  });
+
   it('escapes label values before rendering', async () => {
     prometheusMetricsService.recordRequest('GET', '/api/v1/items/"special"', 500, 10);
 

From 8ef9a9c50d8b8d5ab6ddc7691b8c2bde061a0ce6 Mon Sep 17 00:00:00 2001
From: Aboyeji-Isaac <141873116+Aboyeji-Isaac@users.noreply.github.com>
Date: Tue, 27 Oct 2026 11:22:42 +0100
Subject: [PATCH 37/37] test(metrics): assert unavailable lag omits sample line

---
 tests/unit/prometheusMetricsService.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unit/prometheusMetricsService.test.ts b/tests/unit/prometheusMetricsService.test.ts
index 33c5f91..fc6217d 100644
--- a/tests/unit/prometheusMetricsService.test.ts
+++ b/tests/unit/prometheusMetricsService.test.ts
@@ -129,7 +129,7 @@ describe('PrometheusMetricsService', () => {
     expect(output).toContain('swiftchain_mongodb_pool_stats_available 0');
     expect(output).toContain('swiftchain_redis_healthy 0');
     expect(output).toContain('swiftchain_indexer_lag_available 0');
-    expect(output).not.toContain('swiftchain_indexer_lag_ledgers ');
+    expect(output.split('\n').some((line) => line.startsWith('swiftchain_indexer_lag_ledgers '))).toBe(false);
   });
 
   it('handles Redis and indexer probe errors without breaking exposition output', async () => {