feat: Fix MAL, dynamic requests

Eltik · Eltik · commit fa91fd021396 · 2025-02-14T12:13:11.000-08:00
diff --git a/anify-backend/src/mappings/impl/meta/impl/mal.ts b/anify-backend/src/mappings/impl/meta/impl/mal.ts
@@ -31,27 +31,24 @@ export default class MALMeta extends MetaProvider {
         return results;
     }
 
-    private async fetchResults(query: string, type: MediaType, proxyURL?: string): Promise<IProviderResult[] | undefined> {
+    private async fetchResults(query: string, type: MediaType): Promise<IProviderResult[] | undefined> {
         const results: IProviderResult[] = [];
 
-        const requestConfig: IRequestConfig = {};
-        if (proxyURL) {
-            requestConfig.proxy = proxyURL;
-        }
-
-        requestConfig.validateResponse = async (response) => {
-            if (!response.ok) return false;
-            try {
-                const data = await response.text();
-                const $ = load(data);
-                return $("div.js-categories-seasonal table tr").length > 0;
-            } catch {
-                return false;
-            }
-        };
-
         const url = `${this.url}/${type === MediaType.ANIME ? "anime" : "manga"}.php?q=${query}&c[]=a&c[]=b&c[]=c&c[]=f&c[]=d&c[]=e&c[]=g`;
-        const data = await (await this.request(url, requestConfig)).text();
+        const data = await (
+            await this.request(url, {
+                validateResponse: async (response) => {
+                    if (!response.ok) return false;
+                    try {
+                        const data = await response.text();
+                        const $ = load(data);
+                        return $("div.js-categories-seasonal table tr").length > 0;
+                    } catch {
+                        return false;
+                    }
+                },
+            })
+        ).text();
         const $ = load(data);
 
         const searchResults = $("div.js-categories-seasonal table tr").first();
@@ -72,7 +69,20 @@ export default class MALMeta extends MetaProvider {
 
             promises.push(
                 new Promise(async (resolve) => {
-                    const data = await (await this.request(`${this.url}/${type === MediaType.ANIME ? "anime" : "manga"}/${id}`, requestConfig)).text();
+                    const data = await (
+                        await this.request(`${this.url}/${type === MediaType.ANIME ? "anime" : "manga"}/${id}`, {
+                            validateResponse: async (response) => {
+                                if (!response.ok) return false;
+                                try {
+                                    const data = await response.text();
+                                    const $ = load(data);
+                                    return $("title").text().includes("MyAnimeList");
+                                } catch {
+                                    return false;
+                                }
+                            },
+                        })
+                    ).text();
                     const $$ = load(data);
 
                     const published =
diff --git a/anify-backend/src/mappings/index.ts b/anify-backend/src/mappings/index.ts
@@ -18,10 +18,6 @@ export const ANIME_PROVIDERS = [
         const { default: AnimePahe } = await import("./impl/anime/impl/animepahe");
         return new AnimePahe();
     },
-    async () => {
-        const { default: GogoAnime } = await import("./impl/anime/impl/gogoanime");
-        return new GogoAnime();
-    },
     async () => {
         const { default: HiAnime } = await import("./impl/anime/impl/hianime");
         return new HiAnime();
diff --git a/anify-backend/src/proxies/impl/request/customRequest.ts b/anify-backend/src/proxies/impl/request/customRequest.ts
@@ -13,6 +13,14 @@ const errorTracker = new Map<
     }
 >();
 
+// Track rotation state
+const rotationState = {
+    isRotating: false,
+    lastRotationStart: 0,
+    rotationAttempts: new Map<string, number>(),
+    maxAttempts: 3,
+};
+
 const ROTATION_THRESHOLD = 5;
 const ROTATION_COOLDOWN = 60000;
 const STATUS_CODE_WEIGHTS: Record<number, number> = {
@@ -29,8 +37,33 @@ const MIN_OCCURRENCES: Record<number, number> = {
     999: 2, // Need couple timeouts
 };
 
+// Dynamic backoff calculation based on recent errors and rotation history
+function calculateBackoff(providerKey: string): number {
+    const attempts = rotationState.rotationAttempts.get(providerKey) || 0;
+    const baseDelay = 1000; // 1 second base
+    const jitter = Math.random() * 500; // Add some randomness
+    return Math.min(baseDelay * Math.pow(1.5, attempts) + jitter, 10000); // Cap at 10 seconds
+}
+
+async function waitForRotation(providerKey: string): Promise<void> {
+    if (!rotationState.isRotating) return;
+
+    const backoff = calculateBackoff(providerKey);
+    await new Promise((resolve) => setTimeout(resolve, backoff));
+
+    // Increment attempt counter
+    const attempts = (rotationState.rotationAttempts.get(providerKey) || 0) + 1;
+    rotationState.rotationAttempts.set(providerKey, attempts);
+
+    // If we've waited too long, reset rotation state
+    if (Date.now() - rotationState.lastRotationStart > 30000) {
+        rotationState.isRotating = false;
+        rotationState.rotationAttempts.clear();
+    }
+}
+
 function shouldRotateIP(errorState: { count: number; lastRotation: number; statusCodes: Map<number, number> }): boolean {
-    if (Date.now() - errorState.lastRotation < ROTATION_COOLDOWN) {
+    if (Date.now() - errorState.lastRotation < ROTATION_COOLDOWN || rotationState.isRotating) {
         return false;
     }
 
@@ -69,12 +102,17 @@ async function makeRequest(url: string, options: RequestInit, timeout: number =
 export async function customRequest(url: string, options: IRequestConfig = {}): Promise<Response | null> {
     const { isChecking, proxy, useGoogleTranslate, timeout, providerType, providerId, maxRetries, validateResponse } = options;
     const retryAttempts = isChecking ? 1 : maxRetries || 3;
+    const providerKey = `${providerType}-${providerId}`;
 
     // Try different request strategies in sequence
     for (let attempt = 1; attempt <= retryAttempts; attempt++) {
+        // Wait if IP rotation is in progress
+        if (rotationState.isRotating) {
+            await waitForRotation(providerKey);
+        }
+
         // 1. Try WireGuard proxy first if enabled
         if (!useGoogleTranslate && !isChecking && env.USE_WIREGUARD) {
-            const providerKey = `${providerType}-${providerId}`;
             const errorState = errorTracker.get(providerKey) || {
                 count: 0,
                 lastRotation: 0,
@@ -83,16 +121,25 @@ export async function customRequest(url: string, options: IRequestConfig = {}):
 
             if (shouldRotateIP(errorState)) {
                 try {
+                    rotationState.isRotating = true;
+                    rotationState.lastRotationStart = Date.now();
+
                     await wireguardProxyManager.rotate();
+
                     errorTracker.set(providerKey, {
                         count: 0,
                         lastRotation: Date.now(),
                         statusCodes: new Map(),
                     });
+
                     console.log(`Rotated IP for provider ${providerKey} due to error threshold`);
-                    await new Promise((resolve) => setTimeout(resolve, 1000));
+
+                    // Clear rotation state after successful rotation
+                    rotationState.isRotating = false;
+                    rotationState.rotationAttempts.delete(providerKey);
                 } catch (error) {
                     console.warn("Failed to rotate WireGuard IP:", error);
+                    rotationState.isRotating = false;
                 }
             }
 
@@ -165,7 +212,22 @@ export async function customRequest(url: string, options: IRequestConfig = {}):
         }
     }
 
-    // If all strategies failed, log warning and return null
-    console.warn(`All request strategies failed for ${url}. Provider: ${providerId}/${providerType}`);
+    // If all strategies failed, update error tracking and return null
+    if (providerType && providerId) {
+        const errorState = errorTracker.get(providerKey) || {
+            count: 0,
+            lastRotation: 0,
+            statusCodes: new Map(),
+        };
+
+        // Add to error count and track as a 503 error (service unavailable)
+        const currentCount = errorState.statusCodes.get(503) || 0;
+        errorState.statusCodes.set(503, currentCount + 1);
+        errorState.count++;
+        errorTracker.set(providerKey, errorState);
+
+        console.warn(`All request strategies failed for ${url}. Provider: ${providerId}/${providerType}`);
+    }
+
     return null;
 }
diff --git a/anify-backend/src/types/impl/mappings/impl/mediaProvider.ts b/anify-backend/src/types/impl/mappings/impl/mediaProvider.ts
@@ -4,6 +4,19 @@ import type { IRequestConfig } from "../../proxies";
 import { selectProxy, proxyToUrl } from "../../../../proxies/impl/manager";
 import { customRequest } from "../../../../proxies/impl/request/customRequest";
 
+export class RequestError extends Error {
+    constructor(
+        message: string,
+        public readonly url: string,
+        public readonly providerId: string,
+        public readonly providerType: string,
+        public readonly statusCode?: number,
+    ) {
+        super(message);
+        this.name = "RequestError";
+    }
+}
+
 export abstract class MediaProvider {
     private static limiterMap: Map<string, Bottleneck> = new Map();
 
@@ -20,8 +33,8 @@ export abstract class MediaProvider {
 
     /**
      * Queued request function that respects this.rateLimit (seconds/10).
-     * Returns Response if successful, null if request failed but was handled gracefully.
-     * Throws an error only for unexpected failures that should halt execution.
+     * Returns Response if successful, throws RequestError for handled failures.
+     * Throws other errors only for unexpected failures that should halt execution.
      */
     async request(url: string, config: IRequestConfig = {}, proxyRequest: boolean = false): Promise<Response> {
         if (!MediaProvider.limiterMap.has(this.id)) {
@@ -59,16 +72,33 @@ export abstract class MediaProvider {
 
                 const result = await customRequest(url, finalConfig);
                 if (!result) {
-                    throw new Error(`Request failed for ${url}`);
+                    throw new RequestError(`Request failed after all retry attempts`, url, this.id, this.providerType);
                 }
                 return result;
             });
 
             return response;
         } catch (error) {
-            // Log the error but throw a standardized error to maintain type safety
-            console.error(`Error in request for ${url} (${this.id}/${this.providerType}):`, error);
-            throw new Error(`Failed to fetch ${url}`);
+            if (error instanceof RequestError) {
+                // Log the handled error but don't halt execution
+                console.warn(`Request failed for ${url} (${this.id}/${this.providerType}):`, error.message);
+
+                // Return an empty 204 response instead of null
+                return new Response(null, {
+                    status: 204,
+                    statusText: "No Content - Request Failed",
+                    headers: {
+                        "X-Error-Type": "RequestError",
+                        "X-Error-Message": error.message,
+                        "X-Provider-Id": this.id,
+                        "X-Provider-Type": this.providerType,
+                    },
+                });
+            }
+
+            // For unexpected errors, throw a standardized error
+            console.error(`Unexpected error in request for ${url} (${this.id}/${this.providerType}):`, error);
+            throw new RequestError(`Failed to fetch ${url}`, url, this.id, this.providerType);
         }
     }
 }
