fix(api): pause keyed Hypixel requests until a 429's reset

apps/api/src/hypixel/hypixel.service.ts

@@ -22,10 +22,18 @@ import { Logger } from "@statsify/logger";
 import { Observable, catchError, lastValueFrom, map, of, tap, throwError } from "rxjs";
 import type { APIData } from "@statsify/util";
 
+const DEFAULT_RATE_LIMIT_SECONDS = 60;
+
+const parseHeader = (value: unknown) => {
+  const parsed = Number.parseInt(String(value), 10);
+  return Number.isNaN(parsed) ? undefined : parsed;
+};
+
 @Injectable()
 export class HypixelService {
   private readonly logger = new Logger("HypixelService");
   private resources = new Map<string, APIData>();
+  private rateLimitedUntil = 0;
 
   public constructor(private readonly httpService: HttpService) {}
 
@@ -89,7 +97,7 @@ export class HypixelService {
       this.request<APIData>("/watchdogstats").pipe(
         map((data) => new Watchdog(data)),
         catchError((err) => {
-          this.logger.error(err);
+          if (!(err instanceof HypixelUnavailableException)) this.logger.error(err);
           return of(null);
         })
       )
@@ -104,7 +112,7 @@ export class HypixelService {
           return null;
         }),
         catchError((err) => {
-          this.logger.error(err);
+          if (!(err instanceof HypixelUnavailableException)) this.logger.error(err);
           return of(null);
         })
       )
@@ -130,6 +138,8 @@ export class HypixelService {
   }
 
   private handleFailure<T>(err: any, notFound: T): Observable<T> {
+    if (err instanceof HypixelUnavailableException) return throwError(() => err);
+
     this.logger.error(err);
 
     // Hypixel rejecting the input means there is nothing to find
@@ -139,7 +149,33 @@ export class HypixelService {
     return throwError(() => new HypixelUnavailableException());
   }
 
+  private startRateLimit(headers: Record<string, unknown>) {
+    const seconds =
+      parseHeader(headers["ratelimit-reset"]) ??
+      parseHeader(headers["retry-after"]) ??
+      DEFAULT_RATE_LIMIT_SECONDS;
+
+    const wasRateLimited = Date.now() < this.rateLimitedUntil;
+    this.rateLimitedUntil = Math.max(this.rateLimitedUntil, Date.now() + seconds * 1000);
+
+    if (!wasRateLimited) {
+      this.logger.warn(`Hypixel is rate limiting requests for the next ${seconds} seconds`);
+    }
+  }
+
+  private rateLimitError() {
+    const seconds = Math.ceil((this.rateLimitedUntil - Date.now()) / 1000);
+    return new HypixelUnavailableException(Math.max(1, seconds));
+  }
+
   private request<T>(url: string, params?: Record<string, unknown>): Observable<T> {
+    // Resources don't need an API key, so the key's rate limit doesn't apply to them
+    const isKeyed = !url.startsWith("/resources/");
+
+    if (isKeyed && Date.now() < this.rateLimitedUntil) {
+      return throwError(() => this.rateLimitError());
+    }
+
     const transaction = Sentry.getCurrentHub().getScope()?.getTransaction();
 
     const child = transaction?.startChild({
@@ -151,16 +187,25 @@ export class HypixelService {
       tap((res) => {
         child?.setHttpStatus(res.status);
         child?.finish();
+
+        if (isKeyed && parseHeader(res.headers["ratelimit-remaining"]) === 0) {
+          this.startRateLimit(res.headers);
+        }
       }),
       map((res) => res.data),
-      catchError((err) =>
-        throwError(
+      catchError((err) => {
+        if (isKeyed && err.response?.status === 429) {
+          this.startRateLimit(err.response.headers);
+          return throwError(() => this.rateLimitError());
+        }
+
+        return throwError(
           () =>
             new Error(`Fetching ${url} failed with reason: ${err.message}`, {
               cause: err,
             })
-        )
-      )
+        );
+      })
     );
   }
 }

locales/en-US/default.json

@@ -370,6 +370,10 @@
       "description": "You need to purchase a higher premium tier to unlock this icon!",
       "title": "Higher Tier Required"
     },
+    "hypixelRateLimited": {
+      "description": "Statsify has reached its Hypixel API limit. Please try again in about {{time}}!",
+      "title": "$t(errors.hypixelUnavailable.title)"
+    },
     "hypixelUnavailable": {
       "description": "The Hypixel API is having trouble right now. Please try again in a few minutes!",
       "title": "Hypixel API Unavailable"

packages/api-client/src/exceptions/hypixel.503.ts

@@ -19,11 +19,18 @@ export class HypixelUnavailableException extends ServiceUnavailableException {
   @ApiProperty()
   public error: string;
 
-  public constructor() {
+  @ApiProperty({
+    required: false,
+    description: "Seconds until Hypixel stops rate limiting Statsify, only set when it is",
+  })
+  public retryAfter?: number;
+
+  public constructor(retryAfter?: number) {
     super({
       statusCode: 503,
       error: "Service Unavailable",
       message: "hypixel",
+      retryAfter,
     });
   }
 }

packages/discord/src/services/api.service.ts

@@ -25,7 +25,7 @@ import {
 import { Color, User } from "@statsify/schemas";
 import { ErrorMessage } from "#util/error.message";
 import { Service } from "typedi";
-import { config, removeFormatting } from "@statsify/util";
+import { config, formatTime, removeFormatting } from "@statsify/util";
 
 type PlayerTag = "username" | "uuid" | "discordId" | "none";
 
@@ -55,7 +55,7 @@ export class ApiService extends StatsifyApiService {
         const error = err.response.data as PlayerNotFoundException;
 
         if (error.message === "player") throw this.missingPlayer(type, tag);
-        if (error.message === "hypixel") throw this.hypixelUnavailable();
+        if (error.message === "hypixel") throw this.hypixelUnavailable(error);
 
         throw this.unknownError();
       });
@@ -74,7 +74,7 @@ export class ApiService extends StatsifyApiService {
       const error = err.response.data as PlayerNotFoundException | SessionNotFoundException;
 
       if (error.message === "player") throw this.missingPlayer(type, tag);
-      if (error.message === "hypixel") throw this.hypixelUnavailable();
+      if (error.message === "hypixel") throw this.hypixelUnavailable(error);
 
       if (error.message === "session") {
         const { displayName } = error as SessionNotFoundException;
@@ -106,7 +106,7 @@ export class ApiService extends StatsifyApiService {
         PlayerNotFoundException;
 
       if (error.message === "player") throw this.missingPlayer(type, tag);
-      if (error.message === "hypixel") throw this.hypixelUnavailable();
+      if (error.message === "hypixel") throw this.hypixelUnavailable(error);
 
       if (error.message === "status") {
         const { displayName } = error as StatusNotFoundException;
@@ -183,7 +183,7 @@ export class ApiService extends StatsifyApiService {
       }
 
       if (error.message === "player") throw this.missingPlayer(playerType, input);
-      if (error.message === "hypixel") throw this.hypixelUnavailable();
+      if (error.message === "hypixel") throw this.hypixelUnavailable(error);
 
       throw this.unknownError();
     });
@@ -198,7 +198,7 @@ export class ApiService extends StatsifyApiService {
       if (!err.response || !err.response.data) throw this.unknownError();
 
       const error = err.response.data as HypixelUnavailableException;
-      if (error.message === "hypixel") throw this.hypixelUnavailable();
+      if (error.message === "hypixel") throw this.hypixelUnavailable(error);
 
       if ((err.response.data as PlayerNotFoundException).statusCode === 404) return null;
       throw new ErrorMessage("errors.leaderboardNotFound");
@@ -240,7 +240,7 @@ export class ApiService extends StatsifyApiService {
   public override resetPlayerSession(tag: string) {
     return super.resetPlayerSession(tag).catch((err: AxiosError) => {
       const error = err.response?.data as HypixelUnavailableException | undefined;
-      if (error?.message === "hypixel") throw this.hypixelUnavailable();
+      if (error?.message === "hypixel") throw this.hypixelUnavailable(error);
 
       throw err;
     });
@@ -334,8 +334,15 @@ export class ApiService extends StatsifyApiService {
     return new ErrorMessage("errors.unknown");
   }
 
-  public hypixelUnavailable() {
-    return new ErrorMessage("errors.hypixelUnavailable");
+  public hypixelUnavailable({ retryAfter }: HypixelUnavailableException) {
+    if (!retryAfter) return new ErrorMessage("errors.hypixelUnavailable");
+
+    const time = formatTime(retryAfter * 1000, { short: false, entries: 1 });
+
+    return new ErrorMessage(
+      (t) => t("errors.hypixelRateLimited.title"),
+      (t) => t("errors.hypixelRateLimited.description", { time })
+    );
   }
 
   private isDiscordId(tag: string) {