diff --git a/CHANGELOG.md b/CHANGELOG.md index a4e3c98..00385bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - **Streaming Status Warning**: Added ⚠️ Streaming Status section to README documenting which endpoints work vs which are blocked by upstream anti-bot protection -- **Streaming Fix Solutions**: Added SCRAPER_API_KEY and FLARESOLVERR_URL environment variable solutions to README for bypassing Cloudflare bot protection +- **Proxy Helper**: New `src/helper/proxy.helper.js` — unified proxy support for ScraperAPI and FlareSolverr +- **ScraperAPI Integration**: Routes requests through ScraperAPI premium residential IPs when direct requests fail +- **FlareSolverr Integration**: Routes requests through self-hosted FlareSolverr browser when direct requests fail +- **Proxy Status Endpoint**: `GET /api/proxy/status` — shows configured proxy providers and their status +- **Auto Proxy Fallback**: `fetchWithMirror` now automatically falls back to configured proxies when all mirrors fail +- **Cloudflare Detection**: Detects Cloudflare challenge pages and triggers proxy fallback - **Upstream Error Detection**: Added detection of upstream API error responses (`status >= 400`) in `streamInfo.extractor.js` — returns HTTP 502 with descriptive error instead of silently returning null URL - **Cookie-Aware Session Establishment**: Added session cookie fetching in `extractStreamInfo` and `extractServerList` — fetches watch/home page to capture `set-cookie` headers before AJAX requests - **Optional Slug Parameter**: `getStreamInfo` controller now accepts optional `?slug=` query parameter for session establishment @@ -19,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **README**: Updated version badge from 2.2.1 to 2.3.0 - **README**: Added streaming status warning with solutions table - **Docs**: Updated index.md, endpoints.md, streaming.md with streaming status warnings and v2.3.0 references +- **Mirror Helper**: `fetchWithMirror` now includes proxy fallback and Cloudflare detection ### Fixed - **CRITICAL**: Fixed `/api/stream` returning `null` URL — upstream AJAX endpoints (`/ajax/server?get=`, `/ajax/server/list?servers=`) require session cookies; now properly detects upstream 500 errors and returns HTTP 502 with error message diff --git a/src/helper/mirror.helper.js b/src/helper/mirror.helper.js index d45c2db..7fbbb11 100644 --- a/src/helper/mirror.helper.js +++ b/src/helper/mirror.helper.js @@ -19,6 +19,7 @@ import axios from "axios"; import { headers } from "../configs/header.config.js"; import { getCache, setCache } from "./cache.helper.js"; +import { fetchWithProxy, getProxyStatus } from "./proxy.helper.js"; // ══════════════════════════════════════════════════════════════ // MIRROR CONFIGURATION @@ -144,17 +145,18 @@ function buildUrl(path, baseUrl = workingMirror) { // ---- FEATURE: Resilient Fetch with Mirror Fallback ---- /** - * Fetch with automatic mirror fallback + * Fetch with automatic mirror fallback and proxy support * @param {string} path - URL path to fetch * @param {object} options - Additional options - * @returns {Promise<{data: string, mirror: string}>} + * @returns {Promise<{data: string, mirror: string, proxy: string}>} */ async function fetchWithMirror(path, options = {}) { const { timeout = 15000, retries = 2, returnType = "text", - headers: customHeaders = {} + headers: customHeaders = {}, + useProxy = true, } = options; // Merge default headers with custom headers @@ -182,6 +184,7 @@ async function fetchWithMirror(path, options = {}) { mirrorsToTry.push(...ALL_MIRRORS.map(m => m.url)); } + // Try direct requests first for (const mirror of mirrorsToTry) { const url = buildUrl(path, mirror); @@ -194,6 +197,25 @@ async function fetchWithMirror(path, options = {}) { }); if (response.status === 200) { + // Check for Cloudflare challenge + const data = response.data; + if (typeof data === "string" && ( + data.includes("Just a moment...") || + data.includes("Checking your browser") || + data.includes("Enable JavaScript") + )) { + throw new Error("Cloudflare challenge detected"); + } + + // Check for upstream error + let parsed = data; + if (typeof data === "string") { + try { parsed = JSON.parse(data); } catch { /* not JSON */ } + } + if (parsed && typeof parsed === "object" && parsed.status && Number(parsed.status) >= 400) { + throw new Error(`Upstream error (${parsed.status}): ${parsed.result || "Bad request"}`); + } + // Update working mirror if (mirror !== workingMirror) { workingMirror = mirror; @@ -202,7 +224,8 @@ async function fetchWithMirror(path, options = {}) { } return { data: response.data, - mirror + mirror, + proxy: "direct" }; } @@ -230,6 +253,27 @@ async function fetchWithMirror(path, options = {}) { } } + // All direct requests failed — try proxy if configured + if (useProxy) { + const proxyStatus = getProxyStatus(); + if (proxyStatus.anyEnabled) { + console.log(`[MIRROR] All mirrors failed, trying proxy...`); + try { + const proxyResult = await fetchWithProxy( + buildUrl(path, mirrorsToTry[0]), + { requestHeaders, timeout: timeout * 2, returnType } + ); + return { + data: proxyResult.data, + mirror: mirrorsToTry[0], + proxy: proxyResult.proxy, + }; + } catch (proxyError) { + console.log(`[MIRROR] Proxy also failed: ${proxyError.message}`); + } + } + } + throw new Error(`All mirrors failed. Last error: ${lastError?.message}`); } @@ -265,6 +309,7 @@ export { resetMirrorCache, buildUrl, getMirrorStatus, + getProxyStatus, ALL_MIRRORS }; // ══════════════════════════════════════════════════════════════ END: mirror.helper.js diff --git a/src/helper/proxy.helper.js b/src/helper/proxy.helper.js new file mode 100644 index 0000000..9038b1d --- /dev/null +++ b/src/helper/proxy.helper.js @@ -0,0 +1,211 @@ +/* + * ======= • ======= • ======= • ======= • =======• ======= + * AniKotoAPI — proxy.helper.js + * Repository: https://github.com/Shineii86/AniKotoAPI + * + * @description + * Proxy helper for bypassing Cloudflare bot protection. + * Supports ScraperAPI (premium) and FlareSolverr (self-hosted). + * Automatically routes requests through proxy when direct requests fail. + * + * @exports + * fetchWithProxy, getProxyStatus + * + * @author Shinei Nouzen + * @license MIT + * ======= • ======= • ======= • ======= • =======• ======= + */ + +import axios from "axios"; +import { headers } from "../configs/header.config.js"; + +// ══════════════════════════════════════════════════════════════ +// PROXY CONFIGURATION +// ══════════════════════════════════════════════════════════════ + +const SCRAPER_API_KEY = process.env.SCRAPER_API_KEY; +const FLARESOLVERR_URL = process.env.FLARESOLVERR_URL; + +// ScraperAPI endpoint +const SCRAPER_API_URL = "http://api.scraperapi.com"; + +// ══════════════════════════════════════════════════════════════ +// PROXY STATUS +// ══════════════════════════════════════════════════════════════ + +// ---- FEATURE: Get proxy configuration status ---- +/** + * Returns current proxy configuration status + * @returns {Object} Proxy status with enabled/disabled and available methods + */ +function getProxyStatus() { + return { + scraperApi: { + enabled: !!SCRAPER_API_KEY, + configured: !!SCRAPER_API_KEY, + }, + flaresolverr: { + enabled: !!FLARESOLVERR_URL, + configured: !!FLARESOLVERR_URL, + url: FLARESOLVERR_URL || null, + }, + anyEnabled: !!SCRAPER_API_KEY || !!FLARESOLVERR_URL, + }; +} + +// ══════════════════════════════════════════════════════════════ +// SCRAPERAPI +// ══════════════════════════════════════════════════════════════ + +// ---- FEATURE: ScraperAPI proxy fetch ---- +/** + * Fetch a URL through ScraperAPI (premium residential IPs) + * @param {string} url - Target URL + * @param {Object} options - Request options + * @returns {Promise<{data: string, proxy: string}>} + */ +async function fetchViaScraperApi(url, options = {}) { + const { timeout = 30000, returnType = "text" } = options; + + const params = { + api_key: SCRAPER_API_KEY, + url: url, + render: "false", + }; + + const response = await axios.get(SCRAPER_API_URL, { + params, + timeout, + responseType: returnType === "text" ? "text" : "json", + }); + + return { + data: response.data, + proxy: "scraperapi", + }; +} + +// ══════════════════════════════════════════════════════════════ +// FLARESOLVERR +// ══════════════════════════════════════════════════════════════ + +// ---- FEATURE: FlareSolverr proxy fetch ---- +/** + * Fetch a URL through FlareSolverr (self-hosted browser) + * @param {string} url - Target URL + * @param {Object} options - Request options + * @returns {Promise<{data: string, proxy: string}>} + */ +async function fetchViaFlareSolverr(url, options = {}) { + const { timeout = 60000, returnType = "text" } = options; + + const response = await axios.post( + `${FLARESOLVERR_URL}/v1`, + { + cmd: "request.get", + url: url, + maxTimeout: timeout, + }, + { + headers: { "Content-Type": "application/json" }, + timeout: timeout + 10000, + } + ); + + if (response.data.status === "ok") { + return { + data: response.data.solution.response, + proxy: "flaresolverr", + }; + } + + throw new Error(`FlareSolverr error: ${response.data.message}`); +} + +// ══════════════════════════════════════════════════════════════ +// UNIFIED PROXY FETCH +// ══════════════════════════════════════════════════════════════ + +// ---- FEATURE: Fetch with automatic proxy fallback ---- +/** + * Fetch a URL with automatic proxy fallback when direct requests fail. + * Tries direct first, then falls back to configured proxies. + * + * @param {string} url - Target URL + * @param {Object} options - Request options + * @param {Object} options.requestHeaders - Custom headers for direct request + * @param {number} options.timeout - Request timeout (default: 15000ms) + * @param {string} options.returnType - Response type: "text" or "json" + * @returns {Promise<{data: string, proxy: string}>} + */ +async function fetchWithProxy(url, options = {}) { + const { + requestHeaders = headers, + timeout = 15000, + returnType = "text", + } = options; + + // 1. Try direct request first + try { + const response = await axios.get(url, { + headers: requestHeaders, + timeout, + responseType: returnType === "text" ? "text" : "json", + }); + + if (response.status === 200) { + // Check if response is a valid result (not a Cloudflare block) + const data = response.data; + if (typeof data === "string" && data.includes("Just a moment...")) { + throw new Error("Cloudflare challenge detected"); + } + if (typeof data === "string" && data.includes("Checking your browser")) { + throw new Error("Cloudflare challenge detected"); + } + + return { data, proxy: "direct" }; + } + } catch (error) { + // If ScraperAPI or FlareSolverr configured, try them + if (!SCRAPER_API_KEY && !FLARESOLVERR_URL) { + throw error; + } + } + + // 2. Try ScraperAPI if configured + if (SCRAPER_API_KEY) { + try { + console.log(`[PROXY] Trying ScraperAPI for: ${url}`); + return await fetchViaScraperApi(url, { timeout, returnType }); + } catch (error) { + console.log(`[PROXY] ScraperAPI failed: ${error.message}`); + } + } + + // 3. Try FlareSolverr if configured + if (FLARESOLVERR_URL) { + try { + console.log(`[PROXY] Trying FlareSolverr for: ${url}`); + return await fetchViaFlareSolverr(url, { timeout, returnType }); + } catch (error) { + console.log(`[PROXY] FlareSolverr failed: ${error.message}`); + } + } + + // 4. All methods failed — try direct one more time with higher timeout + try { + const response = await axios.get(url, { + headers: requestHeaders, + timeout: timeout * 2, + responseType: returnType === "text" ? "text" : "json", + }); + + return { data: response.data, proxy: "direct-retry" }; + } catch (error) { + throw new Error(`All fetch methods failed for ${url}: ${error.message}`); + } +} + +export { fetchWithProxy, getProxyStatus }; + +// ══════════════════════════════════════════════════════════════ END: proxy.helper.js diff --git a/src/routes/apiRoutes.js b/src/routes/apiRoutes.js index 205c75e..5328498 100644 --- a/src/routes/apiRoutes.js +++ b/src/routes/apiRoutes.js @@ -48,7 +48,7 @@ import { getTopAnimeRankings } from "../controllers/topAnimeRankings.controller. import { getRecentlyUpdatedTabs } from "../controllers/recentlyUpdatedTabs.controller.js"; import { getCompletedAnime } from "../controllers/completedAnime.controller.js"; import { categoryRoutes } from "./category.route.js"; -import { getMirrorStatus, resetMirrorCache } from "../helper/mirror.helper.js"; +import { getMirrorStatus, resetMirrorCache, getProxyStatus } from "../helper/mirror.helper.js"; import { getCacheStats } from "../helper/cache.helper.js"; // ══════════════════════════════════════════════════════════════ @@ -604,6 +604,12 @@ app.post("/api/mirrors/reset", (req, res) => { res.json({ success: true, message: "Mirror cache reset" }); }); +// ---- FEATURE: Proxy status (ScraperAPI / FlareSolverr) ---- +app.get("/api/proxy/status", (req, res) => { + const status = getProxyStatus(); + res.json({ success: true, results: status }); +}); + // ---- FEATURE: OpenAPI specification ---- app.get("/api/openapi", (req, res) => { res.json({ @@ -664,6 +670,7 @@ app.get("/api/openapi", (req, res) => { "/top-rankings": { get: { summary: "Top anime rankings (day/week/month)", tags: ["Rankings"] } }, "/recently-updated": { get: { summary: "Recently updated (all/dub/sub tabs)", tags: ["Releases"] } }, "/completed": { get: { summary: "Completed anime series", tags: ["Rankings"] } }, + "/proxy/status": { get: { summary: "Proxy status (ScraperAPI / FlareSolverr)", tags: ["System"] } }, }, tags: [ { name: "Home", description: "Homepage data" },