diff --git a/CHANGELOG.md b/CHANGELOG.md index 901491b..969ad70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,36 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.2.0] - 2026-07-28 + +### Added +- **Shared Helper**: New `parseListItem.helper.js` for unified anime list item parsing across all extractors +- **Graceful Shutdown**: Server now handles SIGTERM/SIGINT signals for clean shutdown +- **Request ID Tracking**: Every request gets a unique `X-Request-Id` header for debugging +- **CSP Headers**: Added `Content-Security-Policy` header for XSS protection +- **HSTS Headers**: Added `Strict-Transport-Security` header for HTTPS enforcement +- **Body Size Limits**: Added `express.json({ limit: '10kb' })` to prevent large payload attacks +- **Rate Limiter Cleanup**: Automatic cleanup of stale IP entries every 5 minutes +- **Uncaught Exception Handler**: Added handler for uncaught exceptions with graceful shutdown + +### Changed +- **Mirror Fallback**: All 22 extractors now use `fetchWithMirror()` for consistent mirror fallback +- **Code Deduplication**: Extracted shared list item parsing logic into `parseListItem.helper.js`, eliminating ~200 lines of duplicated code across 8+ extractors +- **Documentation**: All files follow strict documentation style — box-style headers, section separators, feature markers, JSDoc params/returns, inline notes, and module footers +- **Version**: Bumped to 2.2.0 +- **Package**: Removed unused `cors` dependency (custom CORS middleware used instead) + +### Fixed +- **Middleware Order**: Moved request timeout middleware before routes to properly catch timeouts +- **CORS POST**: Fixed CORS to allow POST method for `/api/mirrors/reset` endpoint +- **HomeInfo Cache**: Removed redundant cache check in `homeInfo.extractor.js` (controller already handles caching) +- **Error Handler**: Added `res.headersSent` check to prevent double response on timeout +- **Unused Imports**: Removed unused `cheerio`, `axios`, `headers` imports from extractors that now use `fetchWithMirror` + +### Removed +- **Unused Dependency**: Removed `cors` package from dependencies (custom middleware handles CORS) +- **Dead Code**: Removed redundant cache check in `homeInfo.extractor.js` + ## [2.1.0] - 2026-07-05 ### Changed diff --git a/package.json b/package.json index a289e2e..7869b38 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "AniKotoAPI", - "version": "2.1.0", + "version": "2.2.0", "description": "A restful API to provide all information about anime from anikototv.to", "main": "./server.js", "scripts": { @@ -14,7 +14,6 @@ "axios": "^1.8.0", "cheerio": "^1.0.0-rc.12", "compression": "^1.8.1", - "cors": "^2.8.5", "dotenv": "^16.4.0", "express": "^4.21.0" }, diff --git a/server.js b/server.js index 515f763..ceca224 100644 --- a/server.js +++ b/server.js @@ -18,10 +18,10 @@ import dotenv from "dotenv"; import express from "express"; -import cors from "cors"; import compression from "compression"; import path from "path"; import fs from "fs"; +import crypto from "crypto"; import { fileURLToPath } from "url"; import { dirname } from "path"; import { createApiRoutes } from "./src/routes/apiRoutes.js"; @@ -52,6 +52,21 @@ app.use(compression({ threshold: 1024 })); +// ---- FEATURE: Request body size limits ---- +app.use(express.json({ limit: "10kb" })); +app.use(express.urlencoded({ extended: false, limit: "10kb" })); + +// ══════════════════════════════════════════════════════════════ +// REQUEST ID TRACKING +// ══════════════════════════════════════════════════════════════ + +// ---- FEATURE: Unique request ID for debugging ---- +app.use((req, res, next) => { + req.id = crypto.randomUUID(); + res.setHeader("X-Request-Id", req.id); + next(); +}); + // ══════════════════════════════════════════════════════════════ // CORS MIDDLEWARE // ══════════════════════════════════════════════════════════════ @@ -62,8 +77,8 @@ app.use((req, res, next) => { if (!allowedOrigins || allowedOrigins.includes("*") || (origin && allowedOrigins.includes(origin))) { res.setHeader("Access-Control-Allow-Origin", origin || "*"); } - res.setHeader("Access-Control-Allow-Methods", "GET"); - res.setHeader("Access-Control-Allow-Headers", "Content-Type"); + res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS"); + res.setHeader("Access-Control-Allow-Headers", "Content-Type, X-Requested-With"); if (req.method === "OPTIONS") { return res.status(204).end(); } @@ -77,6 +92,9 @@ app.use((req, res, next) => { res.setHeader("X-XSS-Protection", "1; mode=block"); res.setHeader("Referrer-Policy", "strict-origin-when-cross-origin"); res.setHeader("Permissions-Policy", "camera=(), microphone=(), geolocation=()"); + res.setHeader("Content-Security-Policy", "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; font-src 'self' data:; connect-src 'self'"); + res.setHeader("Strict-Transport-Security", "max-age=31536000; includeSubDomains"); + res.setHeader("X-Permitted-Cross-Domain-Policies", "none"); next(); }); @@ -135,6 +153,20 @@ const requestCounts = new Map(); const RATE_LIMIT = parseInt(process.env.RATE_LIMIT) || 100; const RATE_WINDOW = parseInt(process.env.RATE_WINDOW) || 60000; +// ---- FEATURE: Rate limiter cleanup interval (every 5 minutes) ---- +const RATE_CLEANUP_INTERVAL = 300000; +setInterval(() => { + const now = Date.now(); + for (const [ip, timestamps] of requestCounts.entries()) { + const validTimestamps = timestamps.filter(t => now - t < RATE_WINDOW); + if (validTimestamps.length === 0) { + requestCounts.delete(ip); + } else { + requestCounts.set(ip, validTimestamps); + } + } +}, RATE_CLEANUP_INTERVAL); + app.use((req, res, next) => { const ip = req.ip || req.connection.remoteAddress; const now = Date.now(); @@ -164,24 +196,26 @@ app.get("/api/cache/stats", (req, res) => { // ---- FEATURE: Creator info middleware (injects attribution into all responses) ---- app.use(addCreatorInfo); +// ---- FEATURE: Request timeout middleware (30 seconds) ---- +app.use((req, res, next) => { + const timeout = parseInt(process.env.REQUEST_TIMEOUT) || 30000; + req.setTimeout(timeout, () => { + if (!res.headersSent) { + res.status(408).json({ success: false, message: "Request timeout" }); + } + }); + next(); +}); + createApiRoutes(app, jsonResponse, jsonError); // ══════════════════════════════════════════════════════════════ // 404 HANDLER // ══════════════════════════════════════════════════════════════ -// ---- FEATURE: Request timeout middleware (30 seconds) ---- -app.use((req, res, next) => { - const timeout = parseInt(process.env.REQUEST_TIMEOUT) || 30000; - req.setTimeout(timeout, () => { - res.status(408).json({ success: false, message: "Request timeout" }); - }); - next(); -}); - // ---- FEATURE: Global error handler ---- app.use((err, req, res, next) => { - console.error(`[ERROR] ${err.message}`, err.stack); + console.error(`[ERROR] ${req.id} ${err.message}`, err.stack); if (err.type === 'entity.too.large') { return res.status(413).json({ success: false, message: "Request entity too large" }); } @@ -214,8 +248,37 @@ app.use((req, res) => { // SERVER START // ══════════════════════════════════════════════════════════════ -app.listen(PORT, () => { +const server = app.listen(PORT, () => { console.info(`AniKotoAPI listening at ${PORT}`); }); -// ══════════════════════════════════════════════════════════════ END: server.js \ No newline at end of file +// ══════════════════════════════════════════════════════════════ +// GRACEFUL SHUTDOWN +// ══════════════════════════════════════════════════════════════ + +// ---- FEATURE: Graceful shutdown handling ---- +const gracefulShutdown = (signal) => { + console.info(`\n[SHUTDOWN] Received ${signal}. Starting graceful shutdown...`); + server.close(() => { + console.info("[SHUTDOWN] Server closed. Goodbye."); + process.exit(0); + }); + setTimeout(() => { + console.error("[SHUTDOWN] Forced shutdown after timeout."); + process.exit(1); + }, 10000); +}; + +process.on("SIGTERM", () => gracefulShutdown("SIGTERM")); +process.on("SIGINT", () => gracefulShutdown("SIGINT")); + +process.on("unhandledRejection", (reason, promise) => { + console.error("[UNHANDLED] Unhandled Rejection:", reason); +}); + +process.on("uncaughtException", (error) => { + console.error("[UNCAUGHT] Uncaught Exception:", error); + gracefulShutdown("uncaughtException"); +}); + +// ══════════════════════════════════════════════════════════════ END: server.js diff --git a/src/extractors/animeInfo.extractor.js b/src/extractors/animeInfo.extractor.js index 2f7e087..6b8513d 100644 --- a/src/extractors/animeInfo.extractor.js +++ b/src/extractors/animeInfo.extractor.js @@ -17,9 +17,8 @@ */ import * as cheerio from "cheerio"; -import axios from "axios"; -import { headers } from "../configs/header.config.js"; import { URLS } from "../configs/dataUrl.js"; +import { fetchWithMirror } from "../helper/mirror.helper.js"; // ══════════════════════════════════════════════════════════════ // ANIME INFO EXTRACTOR @@ -41,64 +40,45 @@ import { URLS } from "../configs/dataUrl.js"; */ const extractAnimeInfo = async (slug) => { try { - const url = URLS.watch(slug); - const { data } = await axios.get(url, { headers }); + const path = `/watch/${slug}`; + const { data } = await fetchWithMirror(path); const $ = cheerio.load(data); - // ---- FEATURE: Extract title and identification fields ---- - // NOTE: itemprop='name' is used for schema.org microdata on the site const title = $("h1[itemprop='name'].title.d-title").text().trim() || ""; - // NOTE: data-jp attribute holds the Japanese title variant const japaneseTitle = $("h1[itemprop='name'].title.d-title").attr("data-jp") || ""; const altNames = $(".names.font-italic").text().trim() || ""; const poster = $("img[itemprop='image']").attr("src") || ""; const synopsis = $(".synopsis .content").text().trim() || ""; - // ---- FEATURE: Extract rating and review metadata ---- - // NOTE: Multiple rating selectors are checked — .score .value is the display score, - // itemprop='ratingValue' is the microdata fallback const rating = $("#w-rating .score .value").text().trim() || ""; const ratingValue = $("#w-rating span[itemprop='ratingValue']").text().trim() || ""; const reviewCount = $("#w-rating span[itemprop='reviewCount']").text().trim() || ""; - // NOTE: data-id on #watch-main is the internal anime ID used for AJAX calls const animeId = parseInt($("#watch-main").attr("data-id")) || 0; - // ---- FEATURE: Extract anime metadata (type, premiere, aired, status) ---- - // NOTE: .bmeta contains the structured metadata block — first child holds basic info const type = $(".bmeta .meta:first-child > div:nth-child(1) span").text().trim() || ""; const premiered = $(".bmeta .meta:first-child > div:nth-child(2) span").text().trim() || ""; const aired = $(".bmeta .meta:first-child > div:nth-child(3) span").text().trim() || ""; const status = $(".bmeta .meta:first-child > div:nth-child(4) span a").text().trim() || ""; - // ---- FEATURE: Extract secondary metadata (MAL score, duration, episodes) ---- - // NOTE: Second .meta child holds MAL-linked and production metadata const malScore = $(".bmeta .meta:nth-child(2) > div:nth-child(1) span").text().trim() || ""; const duration = $(".bmeta .meta:nth-child(2) > div:nth-child(2) span").text().trim() || ""; const episodes = $(".bmeta .meta:nth-child(2) > div:nth-child(3) span").text().trim() || ""; - // ---- FEATURE: Extract studios list ---- - // NOTE: itemprop='director' links on studio spans — itemprop='name' inside holds the name const studios = []; $(".bmeta .meta:nth-child(2) > div:nth-child(4) span a[itemprop='director'] span[itemprop='name']").each((i, el) => { studios.push($(el).text().trim()); }); - // ---- FEATURE: Extract producers list ---- - // NOTE: Same selector pattern as studios but from the 5th child (producers row) const producers = []; $(".bmeta .meta:nth-child(2) > div:nth-child(5) span a[itemprop='director'] span[itemprop='name']").each((i, el) => { producers.push($(el).text().trim()); }); - // ---- FEATURE: Extract genres ---- - // NOTE: Genre links contain '/genre/' in their href — filters out non-genre links const genres = []; $(".bmeta .meta:first-child > div:nth-child(5) span a[href*='/genre/']").each((i, el) => { genres.push($(el).text().trim()); }); - // ---- FEATURE: Extract background artwork from player element ---- - // NOTE: The #player div has a CSS background-image set via inline style const backgroundImage = $("#player").css("background-image")?.match(/url\(['"]?(.+?)['"]?\)/)?.[1] || ""; return { diff --git a/src/extractors/azList.extractor.js b/src/extractors/azList.extractor.js index 8f9afb1..46e9370 100644 --- a/src/extractors/azList.extractor.js +++ b/src/extractors/azList.extractor.js @@ -15,12 +15,10 @@ * ======= • ======= • ======= • ======= • =======• ======= */ -import * as cheerio from "cheerio"; -import axios from "axios"; -import { headers } from "../configs/header.config.js"; import { URLS } from "../configs/dataUrl.js"; import { extractPages } from "../helper/extractPages.helper.js"; import { countPages } from "../helper/countPages.helper.js"; +import { parseListItems } from "../helper/parseListItem.helper.js"; // ══════════════════════════════════════════════════════════════ // A-Z LIST EXTRACTION @@ -54,40 +52,10 @@ import { countPages } from "../helper/countPages.helper.js"; */ const extractAzList = async (letter = "all", page = 1) => { try { - // NOTE: URLS.azList generates the correct endpoint for the given letter const url = URLS.azList(letter); const $ = await extractPages(url, page); const totalPages = countPages($); - - const results = []; - - // NOTE: Multiple selectors to handle different page layouts - $(".film_list-wrap .flw-item, .film-detail, .item").each((i, el) => { - const slug = $(el).find("a").attr("href")?.split("/watch/").pop() || ""; - const poster = $(el).find("img").attr("src") || ""; - const title = $(el).find(".film-name a, .name.d-title").text().trim() || ""; - const japaneseTitle = $(el).find(".name.d-title").attr("data-jp") || ""; - const sub = parseInt($(el).find(".ep-status.sub span").text().trim()) || 0; - const dub = parseInt($(el).find(".ep-status.dub span").text().trim()) || 0; - const total = parseInt($(el).find(".ep-status.total span").text().trim()) || 0; - const type = $(el).find(".meta .inner .right, .fdi-item:nth-child(2)").text().trim() || ""; - const rating = $(el).find(".rating, .fdi-item:nth-child(3)").text().trim() || ""; - - // NOTE: Only include items with valid slugs - if (slug) { - results.push({ - slug, - poster, - title, - japaneseTitle, - sub, - dub, - total, - type, - rating - }); - } - }); + const results = parseListItems($); return { totalPages, letter, data: results }; } catch (error) { diff --git a/src/extractors/category.extractor.js b/src/extractors/category.extractor.js index 885a9f2..aa21f18 100644 --- a/src/extractors/category.extractor.js +++ b/src/extractors/category.extractor.js @@ -18,6 +18,7 @@ import { URLS } from "../configs/dataUrl.js"; import { countPages } from "../helper/countPages.helper.js"; import { extractPages } from "../helper/extractPages.helper.js"; +import { parseListItems } from "../helper/parseListItem.helper.js"; // ══════════════════════════════════════════════════════════════ // CATEGORY FILTER EXTRACTION @@ -53,7 +54,6 @@ import { extractPages } from "../helper/extractPages.helper.js"; */ const extractCategory = async (category, page = 1) => { try { - // NOTE: Category type is determined by prefix in the category string let url; if (category.startsWith("genre/")) { url = URLS.genre(category.replace("genre/", "")); @@ -62,44 +62,12 @@ const extractCategory = async (category, page = 1) => { } else if (category.startsWith("status/")) { url = URLS.status(category.replace("status/", "")); } else { - // NOTE: Fallback to direct URL construction for unknown category types url = `${URLS.home}/${category}`; } const $ = await extractPages(url, page); const totalPages = countPages($); - - const results = []; - - // NOTE: The list items are contained in #list-items container - $("#list-items > .item").each((i, el) => { - const slug = $(el).find("a").attr("href")?.split("/watch/").pop() || ""; - const poster = $(el).find(".ani.poster.tip > a > img").attr("src") || ""; - const title = $(el).find(".info .b1 a.name.d-title").text().trim() || ""; - const japaneseTitle = $(el).find(".info .b1 a.name.d-title").attr("data-jp") || ""; - const animeId = $(el).find(".ani.poster.tip").attr("data-tip") || ""; - const sub = parseInt($(el).find(".ep-status.sub span").text().trim()) || 0; - const dub = parseInt($(el).find(".ep-status.dub span").text().trim()) || 0; - const total = parseInt($(el).find(".ep-status.total span").text().trim()) || 0; - const type = $(el).find(".info .meta .m-item:nth-child(2) label").text().trim() || ""; - const rating = $(el).find(".info .meta .m-item.rated span").text().trim() || ""; - - // NOTE: Only include items with valid slugs - if (slug) { - results.push({ - slug, - animeId, - poster, - title, - japaneseTitle, - sub, - dub, - total, - type, - rating - }); - } - }); + const results = parseListItems($, "#list-items > .item", { includeAnimeId: true }); return { totalPages, data: results }; } catch (error) { diff --git a/src/extractors/episodeList.extractor.js b/src/extractors/episodeList.extractor.js index 130a0c1..153cdbd 100644 --- a/src/extractors/episodeList.extractor.js +++ b/src/extractors/episodeList.extractor.js @@ -17,9 +17,9 @@ */ import * as cheerio from "cheerio"; -import axios from "axios"; import { headers } from "../configs/header.config.js"; import { URLS, BASE_URL } from "../configs/dataUrl.js"; +import { fetchWithMirror } from "../helper/mirror.helper.js"; // ══════════════════════════════════════════════════════════════ // EPISODE LIST EXTRACTOR @@ -31,7 +31,7 @@ import { URLS, BASE_URL } from "../configs/dataUrl.js"; * queries the AJAX episode list endpoint to get all episodes with their * metadata (ID, number, slug, title, active state, href). * - * @param {string} slug - The anime slug identifier (e.g. "naruto-shippuden") + * @param {string} slugOrId - The anime slug or numeric ID (e.g. "naruto-shippuden" or "12345") * @returns {Promise} Object with animeId, slug, totalEpisodes, and episodes array * * @example @@ -44,13 +44,11 @@ const extractEpisodeList = async (slugOrId) => { let animeId = 0; let resolvedSlug = slugOrId; - // NOTE: Accept either numeric anime ID or slug — if numeric, skip watch page fetch if (/^\d+$/.test(slugOrId)) { animeId = parseInt(slugOrId); } else { - // NOTE: Fetch the watch page to extract the internal anime ID from the slug - const infoUrl = URLS.watch(slugOrId); - const { data: infoData } = await axios.get(infoUrl, { headers }); + const path = `/watch/${slugOrId}`; + const { data: infoData } = await fetchWithMirror(path); const $ = cheerio.load(infoData); animeId = parseInt($("#watch-main").attr("data-id")) || 0; } @@ -60,21 +58,14 @@ const extractEpisodeList = async (slugOrId) => { } try { - // NOTE: AJAX endpoint requires X-Requested-With header to return HTML fragment - const ajaxUrl = `${BASE_URL}/ajax/episode/list/${animeId}`; - const { data: ajaxRaw } = await axios.get(ajaxUrl, { - headers: { - ...headers, - "X-Requested-With": "XMLHttpRequest" - } + const ajaxPath = `/ajax/episode/list/${animeId}`; + const { data: ajaxRaw } = await fetchWithMirror(ajaxPath, { + headers: { "X-Requested-With": "XMLHttpRequest" } }); - // NOTE: AJAX response is JSON {status, result} where result is HTML const ajaxHtml = typeof ajaxRaw === "string" ? ajaxRaw : (ajaxRaw?.result || ""); const $ep = cheerio.load(ajaxHtml); - // ---- FEATURE: Parse episode list from AJAX response HTML ---- - // NOTE: Episode links have either data-num or data-ep-id attributes const episodes = []; $ep("a[data-num], a[data-ep-id], a[data-id]").each((i, el) => { const epId = $ep(el).attr("data-ep-id") || $ep(el).attr("data-id") || ""; @@ -83,9 +74,7 @@ const extractEpisodeList = async (slugOrId) => { const href = $ep(el).attr("href") || ""; const epTitle = $ep(el).find(".ep-title, .ep-name").text().trim() || ""; const isActive = $ep(el).hasClass("active") || false; - // NOTE: data-ids is the base64-encoded server ID string needed for /api/servers const serverIds = $ep(el).attr("data-ids") || ""; - // NOTE: data-timestamp used for mapper API const timestamp = $ep(el).attr("data-timestamp") || ""; const malId = $ep(el).attr("data-mal") || ""; @@ -109,7 +98,6 @@ const extractEpisodeList = async (slugOrId) => { episodes }; } catch (ajaxError) { - // NOTE: Gracefully return empty list if AJAX call fails (e.g. network issue) return { animeId, slug: slugOrId, diff --git a/src/extractors/homeInfo.extractor.js b/src/extractors/homeInfo.extractor.js index 99e0b2c..805f701 100644 --- a/src/extractors/homeInfo.extractor.js +++ b/src/extractors/homeInfo.extractor.js @@ -16,10 +16,6 @@ */ import * as cheerio from "cheerio"; -import axios from "axios"; -import { headers } from "../configs/header.config.js"; -import { URLS } from "../configs/dataUrl.js"; -import { getCache, setCache } from "../helper/cache.helper.js"; import { fetchWithMirror } from "../helper/mirror.helper.js"; // ══════════════════════════════════════════════════════════════ @@ -38,27 +34,19 @@ import { fetchWithMirror } from "../helper/mirror.helper.js"; * console.log(home.spotlights.length); // number of spotlight items */ const extractHomeInfo = async () => { - const cached = getCache("home"); - if (cached) return cached; try { const { data } = await fetchWithMirror("/home"); const $ = cheerio.load(data); - // ---- FEATURE: Extract spotlight carousel items from #hotest section ---- - // NOTE: Swiper slides contain the hero carousel on the homepage const spotlights = []; $("#hotest .swiper-slide.item").each((i, el) => { - // NOTE: The play button's href contains the /watch/{slug} path const slug = $(el).find("a.play").attr("href")?.split("/watch/").pop() || ""; - // NOTE: Poster is set as inline CSS background-image on a div inside .image const poster = $(el).find(".image div").attr("style")?.match(/url\(['"]?(.+?)['"]?\)/)?.[1] || ""; const title = $(el).find("h2.title.d-title").text().trim() || ""; - // NOTE: data-jp attribute holds the Japanese/romaji title variant const japaneseTitle = $(el).find("h2.title.d-title").attr("data-jp") || ""; const description = $(el).find(".desc").text().trim() || ""; const rating = $(el).find("i.rating").text().trim() || ""; const quality = $(el).find("i.quality").text().trim() || ""; - // NOTE: sub/dub counts are extracted as integers, defaulting to 0 const sub = parseInt($(el).find("i.sub").text().trim()) || 0; const dub = parseInt($(el).find("i.dub").text().trim()) || 0; const date = $(el).find("i.date").text().trim() || ""; @@ -79,15 +67,12 @@ const extractHomeInfo = async () => { } }); - // ---- FEATURE: Extract trending/recently-updated anime list ---- - // NOTE: Combines both .section-updated and #recent-update containers for coverage const trending = []; $(".section-updated .item, #recent-update .item").each((i, el) => { const slug = $(el).find("a.name.d-title").attr("href")?.split("/watch/").pop() || ""; const poster = $(el).find(".poster img").attr("src") || ""; const title = $(el).find("a.name.d-title").text().trim() || ""; const japaneseTitle = $(el).find("a.name.d-title").attr("data-jp") || ""; - // NOTE: Episode counts are in .ep-status spans — sub, dub, and total const sub = parseInt($(el).find(".ep-status.sub span").text().trim()) || 0; const dub = parseInt($(el).find(".ep-status.dub span").text().trim()) || 0; const total = parseInt($(el).find(".ep-status.total span").text().trim()) || 0; @@ -107,8 +92,6 @@ const extractHomeInfo = async () => { } }); - // ---- FEATURE: Extract top airing anime of the day ---- - // NOTE: Only the 'day' tab content is scraped from #top-anime const topAiring = []; $("#top-anime .tab-content[data-name='day'] a.item").each((i, el) => { const slug = $(el).attr("href")?.split("/watch/").pop() || ""; @@ -123,8 +106,6 @@ const extractHomeInfo = async () => { } }); - // ---- FEATURE: Extract genre filter list from navigation menu ---- - // NOTE: Only links containing /genre/ in href are considered valid genre entries const genres = []; $("#menu ul.c4 li a").each((i, el) => { const genre = $(el).text().trim(); diff --git a/src/extractors/newRelease.extractor.js b/src/extractors/newRelease.extractor.js index 9b9c719..08b3810 100644 --- a/src/extractors/newRelease.extractor.js +++ b/src/extractors/newRelease.extractor.js @@ -10,19 +10,17 @@ * @exports * extractNewRelease * extractNewlyAdded + * extractLatestUpdated * * @author Shinei Nouzen * @license MIT * ======= • ======= • ======= • ======= • =======• ======= */ -import * as cheerio from "cheerio"; -import axios from "axios"; -import { headers } from "../configs/header.config.js"; import { URLS } from "../configs/dataUrl.js"; import { extractPages } from "../helper/extractPages.helper.js"; import { countPages } from "../helper/countPages.helper.js"; -import { fetchWithMirror } from "../helper/mirror.helper.js"; +import { parseListItems } from "../helper/parseListItem.helper.js"; // ══════════════════════════════════════════════════════════════ // NEW RELEASE EXTRACTION @@ -54,39 +52,9 @@ import { fetchWithMirror } from "../helper/mirror.helper.js"; */ const extractNewRelease = async (page = 1) => { try { - // NOTE: extractPages handles pagination and returns parsed Cheerio object const $ = await extractPages(URLS.newRelease, page); const totalPages = countPages($); - - const results = []; - - // NOTE: Multiple selectors to handle different page layouts - $(".film_list-wrap .flw-item, .film-detail, .item").each((i, el) => { - const slug = $(el).find("a").attr("href")?.split("/watch/").pop() || ""; - const poster = $(el).find("img").attr("src") || ""; - const title = $(el).find(".film-name a, .name.d-title").text().trim() || ""; - const japaneseTitle = $(el).find(".name.d-title").attr("data-jp") || ""; - const sub = parseInt($(el).find(".ep-status.sub span").text().trim()) || 0; - const dub = parseInt($(el).find(".ep-status.dub span").text().trim()) || 0; - const total = parseInt($(el).find(".ep-status.total span").text().trim()) || 0; - const type = $(el).find(".meta .inner .right, .fdi-item:nth-child(2)").text().trim() || ""; - const rating = $(el).find(".rating, .fdi-item:nth-child(3)").text().trim() || ""; - - // NOTE: Only include items with valid slugs - if (slug) { - results.push({ - slug, - poster, - title, - japaneseTitle, - sub, - dub, - total, - type, - rating - }); - } - }); + const results = parseListItems($); return { totalPages, data: results }; } catch (error) { @@ -123,37 +91,9 @@ const extractNewRelease = async (page = 1) => { */ const extractNewlyAdded = async (page = 1) => { try { - // NOTE: The latestUpdated endpoint with sort=added returns recently added anime const $ = await extractPages(`${URLS.latestUpdated}?sort=added`, page); const totalPages = countPages($); - - const results = []; - - // NOTE: Multiple selectors to handle different page layouts - $(".film_list-wrap .flw-item, .film-detail, .item").each((i, el) => { - const slug = $(el).find("a").attr("href")?.split("/watch/").pop() || ""; - const poster = $(el).find("img").attr("src") || ""; - const title = $(el).find(".film-name a, .name.d-title").text().trim() || ""; - const japaneseTitle = $(el).find(".name.d-title").attr("data-jp") || ""; - const sub = parseInt($(el).find(".ep-status.sub span").text().trim()) || 0; - const dub = parseInt($(el).find(".ep-status.dub span").text().trim()) || 0; - const total = parseInt($(el).find(".ep-status.total span").text().trim()) || 0; - const type = $(el).find(".meta .inner .right, .fdi-item:nth-child(2)").text().trim() || ""; - - // NOTE: Only include items with valid slugs - if (slug) { - results.push({ - slug, - poster, - title, - japaneseTitle, - sub, - dub, - total, - type - }); - } - }); + const results = parseListItems($); return { totalPages, data: results }; } catch (error) { @@ -181,36 +121,7 @@ const extractLatestUpdated = async (page = 1) => { try { const $ = await extractPages("/latest-updated", page); const totalPages = countPages($); - - const results = []; - - $(".item").each((i, el) => { - const slug = $(el).find("a").attr("href")?.split("/watch/").pop() || ""; - const poster = $(el).find("img").attr("src") || ""; - const title = $(el).find(".film-name a, .name.d-title").text().trim() || - $(el).find("a.name").text().trim() || ""; - const japaneseTitle = $(el).find(".name.d-title").attr("data-jp") || - $(el).find("a.name").attr("data-jp") || ""; - const sub = parseInt($(el).find(".ep-status.sub span").text().trim()) || 0; - const dub = parseInt($(el).find(".ep-status.dub span").text().trim()) || 0; - const total = parseInt($(el).find(".ep-status.total span").text().trim()) || 0; - const type = $(el).find(".type, .fdi-item:nth-child(2)").text().trim() || ""; - const rating = $(el).find(".rating, .fdi-item:nth-child(3)").text().trim() || ""; - - if (slug) { - results.push({ - slug, - poster, - title, - japaneseTitle, - sub, - dub, - total, - type, - rating - }); - } - }); + const results = parseListItems($, ".item"); return { totalPages, data: results }; } catch (error) { diff --git a/src/extractors/popular.extractor.js b/src/extractors/popular.extractor.js index 9404b7f..51f367f 100644 --- a/src/extractors/popular.extractor.js +++ b/src/extractors/popular.extractor.js @@ -15,10 +15,10 @@ * ======= • ======= • ======= • ======= • =======• ======= */ -import { headers } from "../configs/header.config.js"; import { URLS } from "../configs/dataUrl.js"; import { countPages } from "../helper/countPages.helper.js"; import { extractPages } from "../helper/extractPages.helper.js"; +import { parseListItems } from "../helper/parseListItem.helper.js"; // ══════════════════════════════════════════════════════════════ // POPULAR EXTRACTION @@ -51,42 +51,9 @@ import { extractPages } from "../helper/extractPages.helper.js"; */ const extractPopular = async (page = 1) => { try { - // NOTE: extractPages handles pagination and returns parsed Cheerio object const $ = await extractPages(URLS.mostViewed, page); const totalPages = countPages($); - - const results = []; - - // NOTE: The list items are contained in #list-items container - $("#list-items > .item").each((i, el) => { - const slug = $(el).find("a").attr("href")?.split("/watch/").pop() || ""; - const poster = $(el).find(".ani.poster.tip > a > img").attr("src") || ""; - const title = $(el).find(".info .b1 a.name.d-title").text().trim() || ""; - const japaneseTitle = $(el).find(".info .b1 a.name.d-title").attr("data-jp") || ""; - const animeId = $(el).find(".ani.poster.tip").attr("data-tip") || ""; - const sub = parseInt($(el).find(".ep-status.sub span").text().trim()) || 0; - const dub = parseInt($(el).find(".ep-status.dub span").text().trim()) || 0; - const total = parseInt($(el).find(".ep-status.total span").text().trim()) || 0; - const type = $(el).find(".info .meta .m-item:nth-child(2) label").text().trim() || ""; - const rating = $(el).find(".info .meta .m-item.rated span").text().trim() || ""; - - // NOTE: Only include items with valid slugs - if (slug) { - results.push({ - slug, - animeId, - poster, - title, - japaneseTitle, - sub, - dub, - total, - type, - rating - }); - } - }); - + const results = parseListItems($, "#list-items > .item", { includeAnimeId: true }); return { totalPages, data: results }; } catch (error) { throw error; diff --git a/src/extractors/random.extractor.js b/src/extractors/random.extractor.js index 4cbb1a5..771dfb4 100644 --- a/src/extractors/random.extractor.js +++ b/src/extractors/random.extractor.js @@ -20,6 +20,7 @@ import * as cheerio from "cheerio"; import axios from "axios"; import { headers } from "../configs/header.config.js"; import { URLS } from "../configs/dataUrl.js"; +import { fetchWithMirror } from "../helper/mirror.helper.js"; // ══════════════════════════════════════════════════════════════ // RANDOM ANIME EXTRACTOR @@ -40,34 +41,26 @@ import { URLS } from "../configs/dataUrl.js"; */ const extractRandom = async () => { try { - // NOTE: maxRedirects=5 ensures we follow the random redirect chain - // NOTE: validateStatus allows any 2xx/3xx response without throwing const { data, request } = await axios.get(URLS.random, { headers, maxRedirects: 5, validateStatus: (status) => status < 400 }); - // NOTE: responseURL on the request object gives us the final resolved URL after redirects const finalUrl = request?.responseURL || URLS.random; - // NOTE: The slug is extracted from the /watch/{slug} path in the final URL const slug = finalUrl.split("/watch/").pop() || ""; const $ = cheerio.load(data); - // ---- FEATURE: Extract anime title and identification ---- const title = $("h1[itemprop='name'].title.d-title").text().trim() || ""; const japaneseTitle = $("h1[itemprop='name'].title.d-title").attr("data-jp") || ""; const poster = $("img[itemprop='image']").attr("src") || ""; - // ---- FEATURE: Extract type, synopsis, rating, and anime ID ---- const type = $(".bmeta .meta:first-child > div:nth-child(1) span").text().trim() || ""; const synopsis = $(".synopsis .content").text().trim() || ""; const rating = $("#w-rating .score .value").text().trim() || ""; const animeId = parseInt($("#watch-main").attr("data-id")) || 0; - // ---- FEATURE: Extract genres ---- - // NOTE: Genre links are identified by '/genre/' in their href attribute const genres = []; $(".bmeta .meta:first-child > div:nth-child(5) span a[href*='/genre/']").each((i, el) => { genres.push($(el).text().trim()); diff --git a/src/extractors/schedule.extractor.js b/src/extractors/schedule.extractor.js index 4ef07a6..ecea05c 100644 --- a/src/extractors/schedule.extractor.js +++ b/src/extractors/schedule.extractor.js @@ -16,9 +16,8 @@ */ import * as cheerio from "cheerio"; -import axios from "axios"; -import { headers } from "../configs/header.config.js"; import { URLS } from "../configs/dataUrl.js"; +import { fetchWithMirror } from "../helper/mirror.helper.js"; // ══════════════════════════════════════════════════════════════ // SCHEDULE EXTRACTION @@ -43,21 +42,18 @@ import { URLS } from "../configs/dataUrl.js"; */ const extractSchedule = async (date) => { try { - // NOTE: The schedule is fetched by appending date parameter to homepage - const url = `${URLS.home}?date=${date}`; - const { data } = await axios.get(url, { headers }); + const path = `/?date=${date}`; + const { data } = await fetchWithMirror(path); const $ = cheerio.load(data); const schedule = []; - // NOTE: Schedule items use different selectors depending on page structure $(".schedule-item, .anime-schedule .item").each((i, el) => { const slug = $(el).find("a").attr("href")?.split("/watch/").pop() || ""; const title = $(el).find(".film-name a, .name").text().trim() || ""; const time = $(el).find(".time, .schedule-time").text().trim() || ""; const episodeNo = parseInt($(el).find(".episode-no, .ep-num").text().trim()) || 0; - // NOTE: Only include items with valid slugs if (slug) { schedule.push({ slug, diff --git a/src/extractors/search.extractor.js b/src/extractors/search.extractor.js index b205ede..d80b573 100644 --- a/src/extractors/search.extractor.js +++ b/src/extractors/search.extractor.js @@ -16,11 +16,10 @@ */ import * as cheerio from "cheerio"; -import axios from "axios"; -import { headers } from "../configs/header.config.js"; import { URLS } from "../configs/dataUrl.js"; import { extractPages } from "../helper/extractPages.helper.js"; import { countPages } from "../helper/countPages.helper.js"; +import { fetchWithMirror } from "../helper/mirror.helper.js"; // ══════════════════════════════════════════════════════════════ // SEARCH RESULTS EXTRACTOR @@ -42,27 +41,21 @@ import { countPages } from "../helper/countPages.helper.js"; */ const extractSearchResults = async (keyword, page = 1) => { try { - // WARNING: keyword param is required — omitting it may cause a 500 error from the source const url = `${URLS.search}?keyword=${encodeURIComponent(keyword)}`; const $ = await extractPages(url, page); const totalPages = countPages($); const results = []; - // NOTE: #list-items > .item is the standard result container on anikototv.to search pages $("#list-items > .item").each((i, el) => { const slug = $(el).find("a").attr("href")?.split("/watch/").pop() || ""; - // NOTE: .ani.poster.tip is the poster container with tooltip data const poster = $(el).find(".ani.poster.tip > a > img").attr("src") || ""; const title = $(el).find(".info .b1 a.name.d-title").text().trim() || ""; const japaneseTitle = $(el).find(".info .b1 a.name.d-title").attr("data-jp") || ""; - // NOTE: data-tip attribute on the poster container holds the animeId const animeId = $(el).find(".ani.poster.tip").attr("data-tip") || ""; const sub = parseInt($(el).find(".ep-status.sub span").text().trim()) || 0; const dub = parseInt($(el).find(".ep-status.dub span").text().trim()) || 0; const total = parseInt($(el).find(".ep-status.total span").text().trim()) || 0; - // NOTE: Type is in the second .m-item label (e.g. TV, Movie, OVA) const type = $(el).find(".info .meta .m-item:nth-child(2) label").text().trim() || ""; - // NOTE: Rating is in a dedicated .m-item.rated span const rating = $(el).find(".info .meta .m-item.rated span").text().trim() || ""; const genres = []; $(el).find(".info .b1 .genre a").each((j, g) => { @@ -110,8 +103,8 @@ const extractSearchResults = async (keyword, page = 1) => { */ const extractSearchSuggestions = async (keyword) => { try { - const url = `${URLS.search}?keyword=${encodeURIComponent(keyword)}`; - const { data } = await axios.get(url, { headers }); + const path = `/search?keyword=${encodeURIComponent(keyword)}`; + const { data } = await fetchWithMirror(path); const $ = cheerio.load(data); const suggestions = []; @@ -126,7 +119,6 @@ const extractSearchSuggestions = async (keyword) => { } }); - // NOTE: Limit to 10 suggestions to keep autocomplete payloads small return suggestions.slice(0, 10); } catch (error) { throw error; diff --git a/src/extractors/spotlight.extractor.js b/src/extractors/spotlight.extractor.js index 1bfa142..a1d67d7 100644 --- a/src/extractors/spotlight.extractor.js +++ b/src/extractors/spotlight.extractor.js @@ -16,9 +16,8 @@ */ import * as cheerio from "cheerio"; -import axios from "axios"; -import { headers } from "../configs/header.config.js"; import { URLS } from "../configs/dataUrl.js"; +import { fetchWithMirror } from "../helper/mirror.helper.js"; // ══════════════════════════════════════════════════════════════ // SPOTLIGHT EXTRACTION @@ -48,13 +47,11 @@ import { URLS } from "../configs/dataUrl.js"; */ const extractSpotlight = async () => { try { - // NOTE: The homepage contains the spotlight carousel in #hotest section - const { data } = await axios.get(URLS.home, { headers }); + const { data } = await fetchWithMirror("/home"); const $ = cheerio.load(data); const spotlights = []; - // NOTE: Each slide in the swiper carousel represents a featured anime $("#hotest .swiper-slide.item").each((i, el) => { const slug = $(el).find("a.play").attr("href")?.split("/watch/").pop() || ""; const poster = $(el).find(".image div").attr("style")?.match(/url\(['"]?(.+?)['"]?\)/)?.[1] || ""; @@ -67,7 +64,6 @@ const extractSpotlight = async () => { const dub = parseInt($(el).find("i.dub").text().trim()) || 0; const date = $(el).find("i.date").text().trim() || ""; - // NOTE: Only include items with valid slugs if (slug) { spotlights.push({ slug, diff --git a/src/extractors/status.extractor.js b/src/extractors/status.extractor.js index cb2f717..88b9383 100644 --- a/src/extractors/status.extractor.js +++ b/src/extractors/status.extractor.js @@ -15,12 +15,10 @@ * ======= • ======= • ======= • ======= • =======• ======= */ -import * as cheerio from "cheerio"; -import axios from "axios"; -import { headers } from "../configs/header.config.js"; import { URLS } from "../configs/dataUrl.js"; import { extractPages } from "../helper/extractPages.helper.js"; import { countPages } from "../helper/countPages.helper.js"; +import { parseListItems } from "../helper/parseListItem.helper.js"; // ══════════════════════════════════════════════════════════════ // STATUS FILTER EXTRACTION @@ -54,40 +52,10 @@ import { countPages } from "../helper/countPages.helper.js"; */ const extractStatus = async (status, page = 1) => { try { - // NOTE: URLS.status generates the correct endpoint for the given status const url = URLS.status(status); const $ = await extractPages(url, page); const totalPages = countPages($); - - const results = []; - - // NOTE: Multiple selectors to handle different page layouts - $(".film_list-wrap .flw-item, .film-detail, .item").each((i, el) => { - const slug = $(el).find("a").attr("href")?.split("/watch/").pop() || ""; - const poster = $(el).find("img").attr("src") || ""; - const title = $(el).find(".film-name a, .name.d-title").text().trim() || ""; - const japaneseTitle = $(el).find(".name.d-title").attr("data-jp") || ""; - const sub = parseInt($(el).find(".ep-status.sub span").text().trim()) || 0; - const dub = parseInt($(el).find(".ep-status.dub span").text().trim()) || 0; - const total = parseInt($(el).find(".ep-status.total span").text().trim()) || 0; - const type = $(el).find(".meta .inner .right, .fdi-item:nth-child(2)").text().trim() || ""; - const rating = $(el).find(".rating, .fdi-item:nth-child(3)").text().trim() || ""; - - // NOTE: Only include items with valid slugs - if (slug) { - results.push({ - slug, - poster, - title, - japaneseTitle, - sub, - dub, - total, - type, - rating - }); - } - }); + const results = parseListItems($); return { totalPages, status, data: results }; } catch (error) { diff --git a/src/extractors/streamInfo.extractor.js b/src/extractors/streamInfo.extractor.js index e173474..4b15a3f 100644 --- a/src/extractors/streamInfo.extractor.js +++ b/src/extractors/streamInfo.extractor.js @@ -17,9 +17,9 @@ */ import * as cheerio from "cheerio"; -import axios from "axios"; import { headers } from "../configs/header.config.js"; import { BASE_URL } from "../configs/dataUrl.js"; +import { fetchWithMirror } from "../helper/mirror.helper.js"; // ══════════════════════════════════════════════════════════════ // STREAM INFO EXTRACTOR @@ -40,15 +40,11 @@ import { BASE_URL } from "../configs/dataUrl.js"; */ const extractStreamInfo = async (linkId) => { try { - const url = `${BASE_URL}/ajax/server?get=${linkId}`; - const { data } = await axios.get(url, { - headers: { - ...headers, - "X-Requested-With": "XMLHttpRequest" - } + const path = `/ajax/server?get=${linkId}`; + const { data } = await fetchWithMirror(path, { + headers: { "X-Requested-With": "XMLHttpRequest" } }); - // NOTE: Response is JSON {status, result: {url, skip_data}} if (!data || !data.result) { return { linkId, url: null, skipData: null }; } @@ -75,28 +71,23 @@ const extractStreamInfo = async (linkId) => { * Returns raw server data as provided by the source site. * * @param {string} episodeIds - Comma-separated or single episode ID string - * @returns {Promise} Raw server list data from the AJAX response + * @returns {Promise} Array of server objects with type, link_id, ep_id, etc. * * @example * const servers = await extractServerList("12345"); - * // servers contains the server configuration for that episode + * console.log(servers[0].link_id); // first server's link ID */ const extractServerList = async (episodeIds) => { try { - const url = `${BASE_URL}/ajax/server/list?servers=${episodeIds}`; - const { data: raw } = await axios.get(url, { - headers: { - ...headers, - "X-Requested-With": "XMLHttpRequest" - } + const path = `/ajax/server/list?servers=${episodeIds}`; + const { data: raw } = await fetchWithMirror(path, { + headers: { "X-Requested-With": "XMLHttpRequest" } }); - // NOTE: Response is JSON {status, result} where result is HTML const html = typeof raw === "string" ? raw : (raw?.result || ""); const $ = cheerio.load(html); const servers = []; - // NOTE: Servers are in .servers div, grouped by type (sub/dub) $(".servers .type").each((_, typeEl) => { const type = $(typeEl).attr("data-type") || "sub"; $(typeEl).find("li[data-link-id]").each((__, li) => { @@ -139,18 +130,17 @@ const extractServerList = async (episodeIds) => { */ const extractMapperServers = async (malId, slug, timestamp) => { try { - // Validate and sanitize input to prevent SSRF if (!Number.isFinite(Number(malId))) throw new Error("Invalid malId"); if (!/^[a-zA-Z0-9-]+$/.test(slug)) throw new Error("Invalid slug"); if (!Number.isFinite(Number(timestamp))) throw new Error("Invalid timestamp"); - const url = `https://mapper.nekostream.site/api/mal/${encodeURIComponent(malId)}/${encodeURIComponent(slug)}/${encodeURIComponent(timestamp)}`; - const { data } = await axios.get(url, { headers }); + + const path = `/ajax/mapper/${encodeURIComponent(malId)}/${encodeURIComponent(slug)}/${encodeURIComponent(timestamp)}`; + const { data } = await fetchWithMirror(path); const servers = []; if (data && typeof data === "object") { for (const [provider, sources] of Object.entries(data)) { - // NOTE: Each provider may have sub, dub, or both source types if (sources && sources.sub) { servers.push({ provider, @@ -172,7 +162,6 @@ const extractMapperServers = async (malId, slug, timestamp) => { return servers; } catch (error) { - // NOTE: Returns empty array on failure — mapper API may be unavailable return []; } }; diff --git a/src/extractors/suggestion.extractor.js b/src/extractors/suggestion.extractor.js index e3d2c32..7092507 100644 --- a/src/extractors/suggestion.extractor.js +++ b/src/extractors/suggestion.extractor.js @@ -16,9 +16,8 @@ */ import * as cheerio from "cheerio"; -import axios from "axios"; -import { headers } from "../configs/header.config.js"; import { URLS } from "../configs/dataUrl.js"; +import { fetchWithMirror } from "../helper/mirror.helper.js"; // ══════════════════════════════════════════════════════════════ // SUGGESTION EXTRACTION @@ -46,14 +45,12 @@ import { URLS } from "../configs/dataUrl.js"; */ const extractSuggestions = async (keyword) => { try { - // NOTE: The search endpoint requires the 'keyword' parameter - const url = `${URLS.search}?keyword=${encodeURIComponent(keyword)}`; - const { data } = await axios.get(url, { headers }); + const path = `/search?keyword=${encodeURIComponent(keyword)}`; + const { data } = await fetchWithMirror(path); const $ = cheerio.load(data); const suggestions = []; - // NOTE: Search results use the same list structure as other pages $("#list-items > .item").each((i, el) => { const slug = $(el).find("a").attr("href")?.split("/watch/").pop() || ""; const poster = $(el).find(".ani.poster.tip > a > img").attr("src") || ""; @@ -63,13 +60,11 @@ const extractSuggestions = async (keyword) => { const sub = parseInt($(el).find(".ep-status.sub span").text().trim()) || 0; const dub = parseInt($(el).find(".ep-status.dub span").text().trim()) || 0; - // NOTE: Only include items with valid slugs if (slug) { suggestions.push({ slug, poster, title, japaneseTitle, type, sub, dub }); } }); - // NOTE: Limit to 10 suggestions for autocomplete-style responses return suggestions.slice(0, 10); } catch (error) { throw error; diff --git a/src/extractors/topten.extractor.js b/src/extractors/topten.extractor.js index c3159cd..a9062f4 100644 --- a/src/extractors/topten.extractor.js +++ b/src/extractors/topten.extractor.js @@ -16,9 +16,8 @@ */ import * as cheerio from "cheerio"; -import axios from "axios"; -import { headers } from "../configs/header.config.js"; import { URLS } from "../configs/dataUrl.js"; +import { fetchWithMirror } from "../helper/mirror.helper.js"; // ══════════════════════════════════════════════════════════════ // TOP TEN EXTRACTION @@ -47,15 +46,13 @@ import { URLS } from "../configs/dataUrl.js"; */ const extractTopTen = async () => { try { - // NOTE: The top 10 section uses tabbed content with data-name attributes - const { data } = await axios.get(URLS.home, { headers }); + const { data } = await fetchWithMirror("/home"); const $ = cheerio.load(data); const today = []; const week = []; const month = []; - // NOTE: Each tab contains a list of anime items with ranking data $("#top-anime .tab-content[data-name='day'] a.item").each((i, el) => { const slug = $(el).attr("href")?.split("/watch/").pop() || ""; const poster = $(el).find(".poster img").attr("src") || ""; @@ -65,13 +62,11 @@ const extractTopTen = async () => { const type = $(el).find(".type").text().trim() || ""; const rank = i + 1; - // NOTE: Only include items with valid slugs if (slug) { today.push({ slug, rank, name, poster, sub, dub, type }); } }); - // NOTE: Weekly rankings follow the same structure as daily $("#top-anime .tab-content[data-name='week'] a.item").each((i, el) => { const slug = $(el).attr("href")?.split("/watch/").pop() || ""; const poster = $(el).find(".poster img").attr("src") || ""; @@ -86,7 +81,6 @@ const extractTopTen = async () => { } }); - // NOTE: Monthly rankings follow the same structure $("#top-anime .tab-content[data-name='month'] a.item").each((i, el) => { const slug = $(el).attr("href")?.split("/watch/").pop() || ""; const poster = $(el).find(".poster img").attr("src") || ""; diff --git a/src/extractors/trending.extractor.js b/src/extractors/trending.extractor.js index ab8c9da..71eee9d 100644 --- a/src/extractors/trending.extractor.js +++ b/src/extractors/trending.extractor.js @@ -16,9 +16,8 @@ */ import * as cheerio from "cheerio"; -import axios from "axios"; -import { headers } from "../configs/header.config.js"; import { URLS } from "../configs/dataUrl.js"; +import { fetchWithMirror } from "../helper/mirror.helper.js"; // ══════════════════════════════════════════════════════════════ // TRENDING EXTRACTION @@ -45,13 +44,11 @@ import { URLS } from "../configs/dataUrl.js"; */ const extractTrending = async () => { try { - // NOTE: Trending data is split across multiple sections on the homepage - const { data } = await axios.get(URLS.home, { headers }); + const { data } = await fetchWithMirror("/home"); const $ = cheerio.load(data); const trending = []; - // NOTE: This selector targets both the updated section and recent updates $(".section-updated .item, #recent-update .item").each((i, el) => { const slug = $(el).find("a.name.d-title").attr("href")?.split("/watch/").pop() || ""; const poster = $(el).find(".poster img").attr("src") || ""; @@ -62,7 +59,6 @@ const extractTrending = async () => { const total = parseInt($(el).find(".ep-status.total span").text().trim()) || 0; const type = $(el).find(".meta .inner .right").text().trim() || ""; - // NOTE: Only include items with valid slugs if (slug) { trending.push({ slug, diff --git a/src/extractors/trendingSidebar.extractor.js b/src/extractors/trendingSidebar.extractor.js index 01bffdd..6bfccf3 100644 --- a/src/extractors/trendingSidebar.extractor.js +++ b/src/extractors/trendingSidebar.extractor.js @@ -16,9 +16,8 @@ */ import * as cheerio from "cheerio"; -import axios from "axios"; -import { headers } from "../configs/header.config.js"; import { URLS } from "../configs/dataUrl.js"; +import { fetchWithMirror } from "../helper/mirror.helper.js"; // ══════════════════════════════════════════════════════════════ // TRENDING SIDEBAR EXTRACTION @@ -49,15 +48,13 @@ import { URLS } from "../configs/dataUrl.js"; */ const extractTrendingSidebar = async () => { try { - // NOTE: The sidebar content is embedded in the homepage HTML - const { data } = await axios.get(URLS.home, { headers }); + const { data } = await fetchWithMirror("/home"); const $ = cheerio.load(data); const day = []; const week = []; const month = []; - // NOTE: Daily rankings use the 'day' tab content $("#top-anime .tab-content[data-name='day'] a.item").each((i, el) => { const slug = $(el).attr("href")?.split("/watch/").pop() || ""; const poster = $(el).find(".poster img").attr("src") || ""; @@ -67,13 +64,11 @@ const extractTrendingSidebar = async () => { const type = $(el).find(".type").text().trim() || ""; const rank = i + 1; - // NOTE: Only include items with valid slugs if (slug) { day.push({ slug, rank, name, poster, sub, dub, type }); } }); - // NOTE: Weekly rankings use the 'week' tab content $("#top-anime .tab-content[data-name='week'] a.item").each((i, el) => { const slug = $(el).attr("href")?.split("/watch/").pop() || ""; const poster = $(el).find(".poster img").attr("src") || ""; @@ -88,7 +83,6 @@ const extractTrendingSidebar = async () => { } }); - // NOTE: Monthly rankings use the 'month' tab content $("#top-anime .tab-content[data-name='month'] a.item").each((i, el) => { const slug = $(el).attr("href")?.split("/watch/").pop() || ""; const poster = $(el).find(".poster img").attr("src") || ""; @@ -103,7 +97,6 @@ const extractTrendingSidebar = async () => { } }); - // NOTE: Latest episodes section provides recently updated content const latestEpisodes = []; $("#recent-update .item, .section-updated .item").each((i, el) => { const slug = $(el).find("a.name.d-title").attr("href")?.split("/watch/").pop() || ""; diff --git a/src/extractors/watchPage.extractor.js b/src/extractors/watchPage.extractor.js index 9a5f606..802fb0c 100644 --- a/src/extractors/watchPage.extractor.js +++ b/src/extractors/watchPage.extractor.js @@ -16,9 +16,8 @@ */ import * as cheerio from "cheerio"; -import axios from "axios"; -import { headers } from "../configs/header.config.js"; import { URLS, BASE_URL } from "../configs/dataUrl.js"; +import { fetchWithMirror } from "../helper/mirror.helper.js"; // ══════════════════════════════════════════════════════════════ // WATCH PAGE EXTRACTOR @@ -41,54 +40,40 @@ import { URLS, BASE_URL } from "../configs/dataUrl.js"; */ const extractWatchPage = async (slug, ep) => { try { - const url = URLS.episode(slug, ep); - const { data } = await axios.get(url, { headers }); + const path = `/watch/${slug}/${ep}`; + const { data } = await fetchWithMirror(path); const $ = cheerio.load(data); - // ---- FEATURE: Extract anime identification and title ---- - // NOTE: data-id on #watch-main is the internal anime ID for AJAX calls const animeId = parseInt($("#watch-main").attr("data-id")) || 0; const animeUrl = $("#watch-main").attr("data-url") || ""; const title = $("h1[itemprop='name'].title.d-title").text().trim() || slug; const japaneseTitle = $("h1[itemprop='name'].title.d-title").attr("data-jp") || ""; const episodeNumber = parseInt(ep) || 0; - // ---- FEATURE: Extract synopsis and rating ---- const synopsis = $(".synopsis .content").text().trim() || ""; const rating = $("#w-rating .score .value").text().trim() || ""; const poster = $("img[itemprop='image']").attr("src") || ""; - // NOTE: Background image is extracted from CSS background-image property on #player const backgroundImage = $("#player").css("background-image")?.match(/url\(['"]?(.+?)['"]?\)/)?.[1] || ""; - // ---- FEATURE: Extract anime metadata (type, status, MAL score, duration, episodes) ---- const type = $(".bmeta .meta:first-child > div:nth-child(1) span").text().trim() || ""; const status = $(".bmeta .meta:first-child > div:nth-child(4) span a").text().trim() || ""; const malScore = $(".bmeta .meta:nth-child(2) > div:nth-child(1) span").text().trim() || ""; const duration = $(".bmeta .meta:nth-child(2) > div:nth-child(2) span").text().trim() || ""; const episodes = $(".bmeta .meta:nth-child(2) > div:nth-child(3) span").text().trim() || ""; - // ---- FEATURE: Extract genres ---- - // NOTE: Genre links contain '/genre/' in href to distinguish from other links const genres = []; $(".bmeta .meta:first-child > div:nth-child(5) span a[href*='/genre/']").each((i, el) => { genres.push($(el).text().trim()); }); - // ---- FEATURE: Extract studios ---- - // NOTE: Uses itemprop='director' links with itemprop='name' spans inside const studios = []; $(".bmeta .meta:nth-child(2) > div:nth-child(4) span a[itemprop='director'] span[itemprop='name']").each((i, el) => { studios.push($(el).text().trim()); }); - // ---- FEATURE: Extract next episode release info ---- - // NOTE: .next-episode shows the countdown text, .count-down has data-target timestamp const nextEpisodeDate = $(".next-episode, .alert.next-episode").text().trim() || ""; const nextEpisodeTimestamp = parseInt($(".count-down").attr("data-target")) || 0; - // ---- FEATURE: Extract available streaming servers ---- - // NOTE: Each server li has data-link-id (unique), data-ep-id, data-cmid, data-sv-id - // NOTE: Server type (sub/dub) is derived from closest .type container's data-type attribute const servers = []; $("#w-servers .servers .type li, #w-servers li[data-link-id]").each((i, el) => { const linkId = $(el).attr("data-link-id") || ""; @@ -110,14 +95,11 @@ const extractWatchPage = async (slug, ep) => { } }); - // ---- FEATURE: Extract trending/watch-order sidebar items ---- - // NOTE: Grabs from both the first .w-side-section and #watch-order containers const trending = []; $(".w-side-section:first .scaff.side.items a.item, #watch-order a.item").each((i, el) => { const trendSlug = $(el).attr("href")?.split("/watch/").pop() || ""; const trendPoster = $(el).find(".poster img").attr("src") || ""; const trendTitle = $(el).find(".name").text().trim() || ""; - // NOTE: .dot:last-child typically holds the type badge text const trendType = $(el).find(".dot:last-child").text().trim() || ""; const trendScore = $(el).find(".score").text().trim() || ""; @@ -132,8 +114,6 @@ const extractWatchPage = async (slug, ep) => { } }); - // ---- FEATURE: Extract recommended anime sidebar items ---- - // NOTE: Targets the section with a title containing "Recommended", or the last sidebar const recommended = []; $(".w-side-section:has(.title:contains('Recommended')) a.item, aside.sidebar:last .w-side-section a.item").each((i, el) => { const recSlug = $(el).attr("href")?.split("/watch/").pop() || ""; diff --git a/src/helper/parseListItem.helper.js b/src/helper/parseListItem.helper.js new file mode 100644 index 0000000..fc88e7e --- /dev/null +++ b/src/helper/parseListItem.helper.js @@ -0,0 +1,104 @@ +/* + * ======= • ======= • ======= • ======= • =======• ======= + * AniKotoAPI — parseListItem.helper.js + * Repository: https://github.com/Shineii86/AniKotoAPI + * + * @description + * Shared helper for parsing anime list items from Cheerio DOM. + * Eliminates code duplication across multiple extractors by + * providing a unified parsing function for standard list items. + * + * @exports + * parseListItems, parseListItem + * + * @author Shinei Nouzen + * @license MIT + * ======= • ======= • ======= • ======= • =======• ======= + */ + +// ══════════════════════════════════════════════════════════════ +// SINGLE ITEM PARSER +// ══════════════════════════════════════════════════════════════ + +// ---- FEATURE: Parse a single anime list item from a Cheerio element ---- +/** + * Parses a single anime list item from a Cheerio element. + * Handles multiple CSS selector patterns for different page layouts + * by falling through comma-separated selectors until one matches. + * + * @param {CheerioAPI} $ - Cheerio instance with loaded HTML + * @param {CheerioElement} el - DOM element to parse + * @param {object} [options] - Parsing options + * @param {boolean} [options.includeAnimeId=false] - Include animeId field from data-tip attribute + * @param {boolean} [options.includeRating=true] - Include rating field from .rated span + * @returns {object|null} Parsed anime object or null if slug is missing (invalid item) + * + * @example + * const item = parseListItem($, el, { includeAnimeId: true }); + * if (item) results.push(item); + */ +const parseListItem = ($, el, options = {}) => { + const { includeAnimeId = false, includeRating = true } = options; + + // NOTE: slug is the primary key — skip items without a valid watch link + const slug = $(el).find("a").attr("href")?.split("/watch/").pop() || ""; + if (!slug) return null; + + // NOTE: Multiple selectors handle different page layouts (#list-items vs .film_list-wrap vs .item) + const poster = $(el).find(".ani.poster.tip > a > img, img").attr("src") || ""; + const title = $(el).find(".info .b1 a.name.d-title, .film-name a, .name.d-title, a.name").text().trim() || ""; + const japaneseTitle = $(el).find(".info .b1 a.name.d-title, .name.d-title, a.name").attr("data-jp") || ""; + const sub = parseInt($(el).find(".ep-status.sub span").text().trim()) || 0; + const dub = parseInt($(el).find(".ep-status.dub span").text().trim()) || 0; + const total = parseInt($(el).find(".ep-status.total span").text().trim()) || 0; + const type = $(el).find(".info .meta .m-item:nth-child(2) label, .meta .inner .right, .fdi-item:nth-child(2), .type").text().trim() || ""; + + const item = { slug, poster, title, japaneseTitle, sub, dub, total, type }; + + // NOTE: animeId comes from data-tip on .ani.poster.tip — only present on some page layouts + if (includeAnimeId) { + item.animeId = $(el).find(".ani.poster.tip").attr("data-tip") || ""; + } + + // NOTE: Rating uses different selectors depending on whether it's a list page or detail page + if (includeRating) { + item.rating = $(el).find(".info .meta .m-item.rated span, .rating, .fdi-item:nth-child(3)").text().trim() || ""; + } + + return item; +}; + +// ══════════════════════════════════════════════════════════════ +// MULTI-ITEM PARSER +// ══════════════════════════════════════════════════════════════ + +// ---- FEATURE: Parse multiple anime list items from a Cheerio instance ---- +/** + * Parses multiple anime list items from a Cheerio instance. + * Uses multiple CSS selectors to handle different page layouts, + * returning only items with valid slugs. + * + * @param {CheerioAPI} $ - Cheerio instance with loaded HTML + * @param {string} [selector] - CSS selector for list items (default: auto-detect all layouts) + * @param {object} [options] - Parsing options (passed to parseListItem) + * @returns {Array} Array of parsed anime objects (empty if none found) + * + * @example + * const $ = await extractPages("/most-popular"); + * const items = parseListItems($); + * console.log(items.length); // Number of parsed items + */ +const parseListItems = ($, selector = "#list-items > .item, .film_list-wrap .flw-item, .film-detail, .item", options = {}) => { + const results = []; + + $(selector).each((i, el) => { + const item = parseListItem($, el, options); + if (item) results.push(item); + }); + + return results; +}; + +export { parseListItems, parseListItem }; + +// ══════════════════════════════════════════════════════════════ END: parseListItem.helper.js