feat: Use CORS proxies

Author: Eltik

anify-backend/package.json

@@ -23,15 +23,13 @@
         "epub-gen-memory": "^1.1.2",
         "eventemitter2": "latest",
         "fastest-levenshtein": "^1.0.16",
-        "https-proxy-agent": "^7.0.6",
         "ioredis": "^5.4.1",
         "jimp": "^0.22.12",
         "lru-cache": "^11.0.2",
         "node-fetch": "^3.3.2",
         "p-limit": "^6.2.0",
         "pdfkit": "^0.13.0",
         "pg": "^8.13.1",
-        "undici": "^7.3.0",
         "zod": "^3.23.8"
     },
     "devDependencies": {

anify-backend/src/env.ts

@@ -49,6 +49,19 @@ const envSchema = z.object({
      */
     DEBUG: booleanFromEnv.optional().default("false"),
 
+    /**
+     * @description Optional, but highly recommended as
+     * Censys is used for scraping proxies.
+     */
+    /**
+     * The API ID for the Censys API.
+     */
+    CENSYS_API_ID: z.string().optional(),
+    /**
+     * The API key for the Censys API.
+     */
+    CENSYS_API_SECRET: z.string().optional(),
+
     /**
      * @description Optional, additional environment variables
      */

anify-backend/src/proxies/impl/manager/impl/check/impl/runProxyChecks.ts

@@ -87,57 +87,89 @@ export const runProxyChecks = async (providers: MediaProvider[], verbose: boolea
         // Check proxies in parallel with concurrency limiting:
         let checkResults: (IProxy | null)[] = [];
         try {
+            const startTime = Date.now();
             checkResults = await Promise.all(
                 proxiesToCheck.map((proxy) =>
                     limit(async () => {
-                        const url = `http://${proxy.ip}:${proxy.port}`;
-                        let isValid = false;
+                        // Add timeout for the entire proxy check process
+                        const proxyCheckPromise = (async () => {
+                            const url = `http://${proxy.ip}:${proxy.port}`;
+                            let isValid = false;
+
+                            try {
+                                // Check if proxy is valid
+                                const timeoutPromise = new Promise<Response>((_, reject) => {
+                                    setTimeout(() => reject(new Error("Request timed out")), 15000);
+                                });
+
+                                const fetchPromise = fetch(`${url}/iscorsneeded`, {
+                                    headers: {
+                                        Origin: "https://anify.tv",
+                                    },
+                                });
+
+                                const response = await Promise.race([fetchPromise, timeoutPromise]);
+
+                                if (response.status === 200 && (await response.text()) === "no") {
+                                    const providerCheckPromise = provider.proxyCheck(url);
+                                    const providerTimeoutPromise = new Promise<boolean>((_, reject) => {
+                                        setTimeout(() => reject(new Error("Provider check timed out")), 15000);
+                                    });
+
+                                    try {
+                                        const check = await Promise.race([providerCheckPromise, providerTimeoutPromise]);
+                                        isValid = check ?? false;
+                                    } catch (error) {
+                                        if (env.DEBUG && verbose) {
+                                            console.error(`Provider check failed for ${url}: ${error instanceof Error ? error.message : String(error)}`);
+                                        }
+                                        isValid = false;
+                                    }
+                                } else {
+                                    isValid = false;
+                                }
+                            } catch {
+                                isValid = false;
+                            }
+
+                            // Update progress counters
+                            checkedCount += 1;
+                            if (isValid) {
+                                validCount += 1;
+                            } else {
+                                invalidCount += 1;
+                            }
 
-                        try {
-                            isValid = (await provider.proxyCheck(url)) ?? false;
-                        } catch (error) {
-                            // Log the error but don't let it crash the script
                             if (env.DEBUG && verbose) {
-                                console.error(`Error checking proxy ${url}: ${error instanceof Error ? error.message : String(error)}`);
+                                throttledProgressUpdate(
+                                    colors.yellow(`Provider: ${provider.providerType} ${provider.id} | ` + `${checkedCount}/${proxiesToCheck.length} checked, ` + `${validCount} valid, ` + `${invalidCount} invalid | ` + `Time elapsed: ${((Date.now() - startTime) / 1000).toFixed(1)}s`),
+                                );
                             }
-                            isValid = false;
-                        }
 
-                        // Update progress counters
+                            return isValid ? proxy : null;
+                        })();
+
+                        // Global timeout for entire proxy check
+                        const globalTimeout = new Promise<null>((_, reject) => {
+                            setTimeout(() => reject(new Error("Global proxy check timeout")), 30000);
+                        });
+
+                        return Promise.race([proxyCheckPromise, globalTimeout]);
+                    }).catch(() => {
                         checkedCount += 1;
-                        if (isValid) {
-                            validCount += 1;
-                        } else {
-                            invalidCount += 1;
-                        }
-
-                        // Throttled progress update:
-                        if (env.DEBUG && verbose) {
-                            throttledProgressUpdate(colors.yellow(`Provider: ${provider.providerType} ${provider.id} | ` + `${checkedCount}/${proxiesToCheck.length} checked, ` + `${validCount} valid, ` + `${invalidCount} invalid`));
-                        }
-
-                        // Return the proxy if valid, otherwise null
-                        return isValid ? proxy : null;
-                    }).catch((error) => {
-                        // Catch any errors that might occur in the limit wrapper
-                        if (env.DEBUG && verbose) {
-                            console.error(`Error in proxy check: ${error instanceof Error ? error.message : String(error)}`);
-                        }
+                        invalidCount += 1;
                         return null;
                     }),
                 ),
-            ).catch((error) => {
-                // Catch any errors in Promise.all
-                if (env.DEBUG && verbose) {
-                    console.error(`Error in proxy batch check: ${error instanceof Error ? error.message : String(error)}`);
-                }
-                return [];
-            });
+            );
         } catch (error) {
             if (env.DEBUG && verbose) {
                 console.error(`Error in proxy batch check: ${error instanceof Error ? error.message : String(error)}`);
             }
             continue;
+        } finally {
+            // Ensure we clear any remaining tasks in the limit queue
+            limit.clearQueue();
         }
 
         // After finishing all checks, print a new line to avoid overwriting

anify-backend/src/proxies/impl/manager/impl/check/index.ts

@@ -1,5 +1,4 @@
 import { MediaProvider } from "../../../../../types/impl/mappings/impl/mediaProvider";
-import proxies from "../scrape";
 import colors from "colors";
 import { runProxyChecks } from "./impl/runProxyChecks";
 import { saveProxies } from "../file/saveProxies";
@@ -63,32 +62,6 @@ export const checkProxies = async (providers: MediaProvider[], verbose: boolean
         }
     }
 
-    // Now scrape for new proxies
-    for (const scrape of Object.values(proxies)) {
-        const newProxies = await scrape();
-
-        if (newProxies.length === 0) {
-            if (env.DEBUG && verbose) {
-                console.log(colors.red("No new proxies found from scraper"));
-            }
-            continue;
-        }
-
-        // Merge new proxies with existing ones, preserving metrics
-        for (const newProxy of newProxies) {
-            const existingProxy = existingProxies.find((p) => p.ip === newProxy.ip && p.port === newProxy.port);
-            if (!existingProxy) {
-                existingProxies.push(newProxy);
-            }
-        }
-
-        await saveProxies(existingProxies);
-
-        if (env.DEBUG && verbose) {
-            console.log(colors.green(`Found ${newProxies.length} new proxies. Total proxies after merge: ${existingProxies.length}`));
-        }
-    }
-
     // Preload the proxies into cache before running checks
     await preloadProxies();
 

anify-backend/src/proxies/impl/manager/impl/file/preloadProxies.ts

@@ -1,7 +1,6 @@
 import { proxyCache } from "../..";
 import { PROVIDERS } from "../../../../../mappings";
 import type { IProxy } from "../../../../../types/impl/proxies";
-import { ProxyType } from "../../../../../types/impl/proxies";
 import { loadJSON } from "../../../helper/loadJSON";
 import { emitter } from "../../../../../events";
 import { Events } from "../../../../../types/impl/events";
@@ -81,8 +80,6 @@ export async function preloadProxies(): Promise<void> {
                 convertedProxies.forEach((typeProxy) => {
                     const existingProxy = proxyCache.proxies.find((p) => p.ip === typeProxy.ip && p.port === typeProxy.port);
                     if (existingProxy) {
-                        // Merge provider metrics and preserve type
-                        existingProxy.type = typeProxy.type || existingProxy.type;
                         existingProxy.providerMetrics = {
                             ...existingProxy.providerMetrics,
                             [provider.id]: typeProxy.providerMetrics[provider.id],
@@ -91,7 +88,6 @@ export async function preloadProxies(): Promise<void> {
                         // Only include the metrics for this specific provider but preserve all other properties
                         const newProxy = {
                             ...typeProxy,
-                            type: typeProxy.type || ProxyType.HTTP, // Ensure type is preserved
                             providerMetrics: {
                                 [provider.id]: typeProxy.providerMetrics[provider.id],
                             },

anify-backend/src/proxies/impl/manager/impl/file/saveProviderProxies.ts

@@ -2,7 +2,6 @@ import { proxyCache } from "../..";
 import { ProviderType } from "../../../../../types";
 import { saveJSON } from "../../../helper/saveJSON";
 import type { IProxy } from "../../../../../types/impl/proxies";
-import { ProxyType } from "../../../../../types/impl/proxies";
 import { env } from "../../../../../env";
 import colors from "colors";
 
@@ -23,7 +22,6 @@ export async function saveProviderProxies(providerType: ProviderType): Promise<v
                 // If this is a new proxy, add it to our final list with proper type
                 finalProxies[key] = {
                     ...proxy,
-                    type: proxy.type || ProxyType.HTTP, // Default to HTTP if type is not set
                     providerMetrics: {
                         [providerId]: {
                             healthScore: 50,
@@ -50,8 +48,6 @@ export async function saveProviderProxies(providerType: ProviderType): Promise<v
                     latencyScore: 50,
                 };
             } else {
-                // If this proxy already exists, update the provider metrics and ensure type is preserved
-                finalProxies[key].type = proxy.type || finalProxies[key].type;
                 finalProxies[key].providerMetrics[providerId] = proxy.providerMetrics?.[providerId] || {
                     healthScore: 50,
                     consecutiveFailures: 0,