mirror of
https://github.com/Shineii86/AniKotoAPI.git
synced 2026-10-22 10:55:10 +00:00
Implement ScraperAPI and FlareSolverr proxy support
- Add proxy.helper.js with unified proxy support - Add ScraperAPI integration (premium residential IPs) - Add FlareSolverr integration (self-hosted browser) - Add proxy fallback to fetchWithMirror() - Add Cloudflare challenge detection - Add GET /api/proxy/status endpoint - Update mirror.helper.js with proxy integration
This commit is contained in:
Shinei Nouzen
+7
-1
@@ -9,7 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Added
|
||||
- **Streaming Status Warning**: Added ⚠️ Streaming Status section to README documenting which endpoints work vs which are blocked by upstream anti-bot protection
|
||||
- **Streaming Fix Solutions**: Added SCRAPER_API_KEY and FLARESOLVERR_URL environment variable solutions to README for bypassing Cloudflare bot protection
|
||||
- **Proxy Helper**: New `src/helper/proxy.helper.js` — unified proxy support for ScraperAPI and FlareSolverr
|
||||
- **ScraperAPI Integration**: Routes requests through ScraperAPI premium residential IPs when direct requests fail
|
||||
- **FlareSolverr Integration**: Routes requests through self-hosted FlareSolverr browser when direct requests fail
|
||||
- **Proxy Status Endpoint**: `GET /api/proxy/status` — shows configured proxy providers and their status
|
||||
- **Auto Proxy Fallback**: `fetchWithMirror` now automatically falls back to configured proxies when all mirrors fail
|
||||
- **Cloudflare Detection**: Detects Cloudflare challenge pages and triggers proxy fallback
|
||||
- **Upstream Error Detection**: Added detection of upstream API error responses (`status >= 400`) in `streamInfo.extractor.js` — returns HTTP 502 with descriptive error instead of silently returning null URL
|
||||
- **Cookie-Aware Session Establishment**: Added session cookie fetching in `extractStreamInfo` and `extractServerList` — fetches watch/home page to capture `set-cookie` headers before AJAX requests
|
||||
- **Optional Slug Parameter**: `getStreamInfo` controller now accepts optional `?slug=` query parameter for session establishment
|
||||
@@ -19,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- **README**: Updated version badge from 2.2.1 to 2.3.0
|
||||
- **README**: Added streaming status warning with solutions table
|
||||
- **Docs**: Updated index.md, endpoints.md, streaming.md with streaming status warnings and v2.3.0 references
|
||||
- **Mirror Helper**: `fetchWithMirror` now includes proxy fallback and Cloudflare detection
|
||||
|
||||
### Fixed
|
||||
- **CRITICAL**: Fixed `/api/stream` returning `null` URL — upstream AJAX endpoints (`/ajax/server?get=`, `/ajax/server/list?servers=`) require session cookies; now properly detects upstream 500 errors and returns HTTP 502 with error message
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
import axios from "axios";
|
||||
import { headers } from "../configs/header.config.js";
|
||||
import { getCache, setCache } from "./cache.helper.js";
|
||||
import { fetchWithProxy, getProxyStatus } from "./proxy.helper.js";
|
||||
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
// MIRROR CONFIGURATION
|
||||
@@ -144,17 +145,18 @@ function buildUrl(path, baseUrl = workingMirror) {
|
||||
|
||||
// ---- FEATURE: Resilient Fetch with Mirror Fallback ----
|
||||
/**
|
||||
* Fetch with automatic mirror fallback
|
||||
* Fetch with automatic mirror fallback and proxy support
|
||||
* @param {string} path - URL path to fetch
|
||||
* @param {object} options - Additional options
|
||||
* @returns {Promise<{data: string, mirror: string}>}
|
||||
* @returns {Promise<{data: string, mirror: string, proxy: string}>}
|
||||
*/
|
||||
async function fetchWithMirror(path, options = {}) {
|
||||
const {
|
||||
timeout = 15000,
|
||||
retries = 2,
|
||||
returnType = "text",
|
||||
headers: customHeaders = {}
|
||||
headers: customHeaders = {},
|
||||
useProxy = true,
|
||||
} = options;
|
||||
|
||||
// Merge default headers with custom headers
|
||||
@@ -182,6 +184,7 @@ async function fetchWithMirror(path, options = {}) {
|
||||
mirrorsToTry.push(...ALL_MIRRORS.map(m => m.url));
|
||||
}
|
||||
|
||||
// Try direct requests first
|
||||
for (const mirror of mirrorsToTry) {
|
||||
const url = buildUrl(path, mirror);
|
||||
|
||||
@@ -194,6 +197,25 @@ async function fetchWithMirror(path, options = {}) {
|
||||
});
|
||||
|
||||
if (response.status === 200) {
|
||||
// Check for Cloudflare challenge
|
||||
const data = response.data;
|
||||
if (typeof data === "string" && (
|
||||
data.includes("Just a moment...") ||
|
||||
data.includes("Checking your browser") ||
|
||||
data.includes("Enable JavaScript")
|
||||
)) {
|
||||
throw new Error("Cloudflare challenge detected");
|
||||
}
|
||||
|
||||
// Check for upstream error
|
||||
let parsed = data;
|
||||
if (typeof data === "string") {
|
||||
try { parsed = JSON.parse(data); } catch { /* not JSON */ }
|
||||
}
|
||||
if (parsed && typeof parsed === "object" && parsed.status && Number(parsed.status) >= 400) {
|
||||
throw new Error(`Upstream error (${parsed.status}): ${parsed.result || "Bad request"}`);
|
||||
}
|
||||
|
||||
// Update working mirror
|
||||
if (mirror !== workingMirror) {
|
||||
workingMirror = mirror;
|
||||
@@ -202,7 +224,8 @@ async function fetchWithMirror(path, options = {}) {
|
||||
}
|
||||
return {
|
||||
data: response.data,
|
||||
mirror
|
||||
mirror,
|
||||
proxy: "direct"
|
||||
};
|
||||
}
|
||||
|
||||
@@ -230,6 +253,27 @@ async function fetchWithMirror(path, options = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
// All direct requests failed — try proxy if configured
|
||||
if (useProxy) {
|
||||
const proxyStatus = getProxyStatus();
|
||||
if (proxyStatus.anyEnabled) {
|
||||
console.log(`[MIRROR] All mirrors failed, trying proxy...`);
|
||||
try {
|
||||
const proxyResult = await fetchWithProxy(
|
||||
buildUrl(path, mirrorsToTry[0]),
|
||||
{ requestHeaders, timeout: timeout * 2, returnType }
|
||||
);
|
||||
return {
|
||||
data: proxyResult.data,
|
||||
mirror: mirrorsToTry[0],
|
||||
proxy: proxyResult.proxy,
|
||||
};
|
||||
} catch (proxyError) {
|
||||
console.log(`[MIRROR] Proxy also failed: ${proxyError.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`All mirrors failed. Last error: ${lastError?.message}`);
|
||||
}
|
||||
|
||||
@@ -265,6 +309,7 @@ export {
|
||||
resetMirrorCache,
|
||||
buildUrl,
|
||||
getMirrorStatus,
|
||||
getProxyStatus,
|
||||
ALL_MIRRORS
|
||||
};
|
||||
// ══════════════════════════════════════════════════════════════ END: mirror.helper.js
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
/*
|
||||
* ======= • ======= • ======= • ======= • =======• =======
|
||||
* AniKotoAPI — proxy.helper.js
|
||||
* Repository: https://github.com/Shineii86/AniKotoAPI
|
||||
*
|
||||
* @description
|
||||
* Proxy helper for bypassing Cloudflare bot protection.
|
||||
* Supports ScraperAPI (premium) and FlareSolverr (self-hosted).
|
||||
* Automatically routes requests through proxy when direct requests fail.
|
||||
*
|
||||
* @exports
|
||||
* fetchWithProxy, getProxyStatus
|
||||
*
|
||||
* @author Shinei Nouzen
|
||||
* @license MIT
|
||||
* ======= • ======= • ======= • ======= • =======• =======
|
||||
*/
|
||||
|
||||
import axios from "axios";
|
||||
import { headers } from "../configs/header.config.js";
|
||||
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
// PROXY CONFIGURATION
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
|
||||
const SCRAPER_API_KEY = process.env.SCRAPER_API_KEY;
|
||||
const FLARESOLVERR_URL = process.env.FLARESOLVERR_URL;
|
||||
|
||||
// ScraperAPI endpoint
|
||||
const SCRAPER_API_URL = "http://api.scraperapi.com";
|
||||
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
// PROXY STATUS
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
|
||||
// ---- FEATURE: Get proxy configuration status ----
|
||||
/**
|
||||
* Returns current proxy configuration status
|
||||
* @returns {Object} Proxy status with enabled/disabled and available methods
|
||||
*/
|
||||
function getProxyStatus() {
|
||||
return {
|
||||
scraperApi: {
|
||||
enabled: !!SCRAPER_API_KEY,
|
||||
configured: !!SCRAPER_API_KEY,
|
||||
},
|
||||
flaresolverr: {
|
||||
enabled: !!FLARESOLVERR_URL,
|
||||
configured: !!FLARESOLVERR_URL,
|
||||
url: FLARESOLVERR_URL || null,
|
||||
},
|
||||
anyEnabled: !!SCRAPER_API_KEY || !!FLARESOLVERR_URL,
|
||||
};
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
// SCRAPERAPI
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
|
||||
// ---- FEATURE: ScraperAPI proxy fetch ----
|
||||
/**
|
||||
* Fetch a URL through ScraperAPI (premium residential IPs)
|
||||
* @param {string} url - Target URL
|
||||
* @param {Object} options - Request options
|
||||
* @returns {Promise<{data: string, proxy: string}>}
|
||||
*/
|
||||
async function fetchViaScraperApi(url, options = {}) {
|
||||
const { timeout = 30000, returnType = "text" } = options;
|
||||
|
||||
const params = {
|
||||
api_key: SCRAPER_API_KEY,
|
||||
url: url,
|
||||
render: "false",
|
||||
};
|
||||
|
||||
const response = await axios.get(SCRAPER_API_URL, {
|
||||
params,
|
||||
timeout,
|
||||
responseType: returnType === "text" ? "text" : "json",
|
||||
});
|
||||
|
||||
return {
|
||||
data: response.data,
|
||||
proxy: "scraperapi",
|
||||
};
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
// FLARESOLVERR
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
|
||||
// ---- FEATURE: FlareSolverr proxy fetch ----
|
||||
/**
|
||||
* Fetch a URL through FlareSolverr (self-hosted browser)
|
||||
* @param {string} url - Target URL
|
||||
* @param {Object} options - Request options
|
||||
* @returns {Promise<{data: string, proxy: string}>}
|
||||
*/
|
||||
async function fetchViaFlareSolverr(url, options = {}) {
|
||||
const { timeout = 60000, returnType = "text" } = options;
|
||||
|
||||
const response = await axios.post(
|
||||
`${FLARESOLVERR_URL}/v1`,
|
||||
{
|
||||
cmd: "request.get",
|
||||
url: url,
|
||||
maxTimeout: timeout,
|
||||
},
|
||||
{
|
||||
headers: { "Content-Type": "application/json" },
|
||||
timeout: timeout + 10000,
|
||||
}
|
||||
);
|
||||
|
||||
if (response.data.status === "ok") {
|
||||
return {
|
||||
data: response.data.solution.response,
|
||||
proxy: "flaresolverr",
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(`FlareSolverr error: ${response.data.message}`);
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
// UNIFIED PROXY FETCH
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
|
||||
// ---- FEATURE: Fetch with automatic proxy fallback ----
|
||||
/**
|
||||
* Fetch a URL with automatic proxy fallback when direct requests fail.
|
||||
* Tries direct first, then falls back to configured proxies.
|
||||
*
|
||||
* @param {string} url - Target URL
|
||||
* @param {Object} options - Request options
|
||||
* @param {Object} options.requestHeaders - Custom headers for direct request
|
||||
* @param {number} options.timeout - Request timeout (default: 15000ms)
|
||||
* @param {string} options.returnType - Response type: "text" or "json"
|
||||
* @returns {Promise<{data: string, proxy: string}>}
|
||||
*/
|
||||
async function fetchWithProxy(url, options = {}) {
|
||||
const {
|
||||
requestHeaders = headers,
|
||||
timeout = 15000,
|
||||
returnType = "text",
|
||||
} = options;
|
||||
|
||||
// 1. Try direct request first
|
||||
try {
|
||||
const response = await axios.get(url, {
|
||||
headers: requestHeaders,
|
||||
timeout,
|
||||
responseType: returnType === "text" ? "text" : "json",
|
||||
});
|
||||
|
||||
if (response.status === 200) {
|
||||
// Check if response is a valid result (not a Cloudflare block)
|
||||
const data = response.data;
|
||||
if (typeof data === "string" && data.includes("Just a moment...")) {
|
||||
throw new Error("Cloudflare challenge detected");
|
||||
}
|
||||
if (typeof data === "string" && data.includes("Checking your browser")) {
|
||||
throw new Error("Cloudflare challenge detected");
|
||||
}
|
||||
|
||||
return { data, proxy: "direct" };
|
||||
}
|
||||
} catch (error) {
|
||||
// If ScraperAPI or FlareSolverr configured, try them
|
||||
if (!SCRAPER_API_KEY && !FLARESOLVERR_URL) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Try ScraperAPI if configured
|
||||
if (SCRAPER_API_KEY) {
|
||||
try {
|
||||
console.log(`[PROXY] Trying ScraperAPI for: ${url}`);
|
||||
return await fetchViaScraperApi(url, { timeout, returnType });
|
||||
} catch (error) {
|
||||
console.log(`[PROXY] ScraperAPI failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Try FlareSolverr if configured
|
||||
if (FLARESOLVERR_URL) {
|
||||
try {
|
||||
console.log(`[PROXY] Trying FlareSolverr for: ${url}`);
|
||||
return await fetchViaFlareSolverr(url, { timeout, returnType });
|
||||
} catch (error) {
|
||||
console.log(`[PROXY] FlareSolverr failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 4. All methods failed — try direct one more time with higher timeout
|
||||
try {
|
||||
const response = await axios.get(url, {
|
||||
headers: requestHeaders,
|
||||
timeout: timeout * 2,
|
||||
responseType: returnType === "text" ? "text" : "json",
|
||||
});
|
||||
|
||||
return { data: response.data, proxy: "direct-retry" };
|
||||
} catch (error) {
|
||||
throw new Error(`All fetch methods failed for ${url}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
export { fetchWithProxy, getProxyStatus };
|
||||
|
||||
// ══════════════════════════════════════════════════════════════ END: proxy.helper.js
|
||||
@@ -48,7 +48,7 @@ import { getTopAnimeRankings } from "../controllers/topAnimeRankings.controller.
|
||||
import { getRecentlyUpdatedTabs } from "../controllers/recentlyUpdatedTabs.controller.js";
|
||||
import { getCompletedAnime } from "../controllers/completedAnime.controller.js";
|
||||
import { categoryRoutes } from "./category.route.js";
|
||||
import { getMirrorStatus, resetMirrorCache } from "../helper/mirror.helper.js";
|
||||
import { getMirrorStatus, resetMirrorCache, getProxyStatus } from "../helper/mirror.helper.js";
|
||||
import { getCacheStats } from "../helper/cache.helper.js";
|
||||
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
@@ -604,6 +604,12 @@ app.post("/api/mirrors/reset", (req, res) => {
|
||||
res.json({ success: true, message: "Mirror cache reset" });
|
||||
});
|
||||
|
||||
// ---- FEATURE: Proxy status (ScraperAPI / FlareSolverr) ----
|
||||
app.get("/api/proxy/status", (req, res) => {
|
||||
const status = getProxyStatus();
|
||||
res.json({ success: true, results: status });
|
||||
});
|
||||
|
||||
// ---- FEATURE: OpenAPI specification ----
|
||||
app.get("/api/openapi", (req, res) => {
|
||||
res.json({
|
||||
@@ -664,6 +670,7 @@ app.get("/api/openapi", (req, res) => {
|
||||
"/top-rankings": { get: { summary: "Top anime rankings (day/week/month)", tags: ["Rankings"] } },
|
||||
"/recently-updated": { get: { summary: "Recently updated (all/dub/sub tabs)", tags: ["Releases"] } },
|
||||
"/completed": { get: { summary: "Completed anime series", tags: ["Rankings"] } },
|
||||
"/proxy/status": { get: { summary: "Proxy status (ScraperAPI / FlareSolverr)", tags: ["System"] } },
|
||||
},
|
||||
tags: [
|
||||
{ name: "Home", description: "Homepage data" },
|
||||
|
||||
Reference in New Issue
Block a user