feat(route): add support for f95zone.to (#21042)
* feat(route): add f95zone routes * fix(route): fix f95zone threads route for folo parse * fix(route): parse f95zone thread guid from [YYYY-MM-DD] in title * fix(route): use update date in thread title as pubDate * fix(route/f95zone): revert image URL processing * fix(route/f95zone): reenable image URL processing for telegram-bot * feat(route/f95zone): support post and thread route * feat(route/f95zone): optimize title * feat(route/f95zone): optimize image display * feat(route/f95zone): optimize image display * feat(route/f95zone): reoptimize img-elements processing * feat(route/f95zone): reset to the original route pattern * feat(route/f95zone): optimize routes * feat(route/f95zone): optimize content processing * feat(route/f95zone): reduce code * fix(route/f95zone): use parseDate for proper date handling * fix(route/f95zone): remove broken radar * fix(route/f95zone): remove invalid radar config and improve docs - Remove radar config from post route (postId is in URL hash, cannot be extracted) - Add empty radar array to prevent Folo parsing error - Add detailed URL format documentation for both routes - Explain how to convert browser URL to subscription URL * fix(route/f95zone): fix invalid example URL * fix(route/f95zone): use internally generated UA * fix(route/f95zone): only fetch the first and the last page * fix(route/f95zone): remove unnecessary sorting logic
This commit is contained in:
wsmbsbbz
GitHub
parent
5ae169a201
commit
dc68ecb875
|
|
@ -102,6 +102,7 @@ type ConfigEnvKeys =
|
|||
| 'EH_STAR'
|
||||
| 'EH_IMG_PROXY'
|
||||
| `EMAIL_CONFIG_${string}`
|
||||
| 'F95ZONE_COOKIE'
|
||||
| 'FANBOX_SESSION_ID'
|
||||
| 'FANFOU_CONSUMER_KEY'
|
||||
| 'FANFOU_CONSUMER_SECRET'
|
||||
|
|
@ -392,6 +393,9 @@ export type Config = {
|
|||
email: {
|
||||
config: Record<string, string | undefined>;
|
||||
};
|
||||
f95zone: {
|
||||
cookie?: string;
|
||||
};
|
||||
fanbox: {
|
||||
session?: string;
|
||||
};
|
||||
|
|
@ -871,6 +875,9 @@ const calculateValue = () => {
|
|||
email: {
|
||||
config: email_config,
|
||||
},
|
||||
f95zone: {
|
||||
cookie: envs.F95ZONE_COOKIE,
|
||||
},
|
||||
fanbox: {
|
||||
session: envs.FANBOX_SESSION_ID,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
import type { Namespace } from '@/types';
|
||||
|
||||
export const namespace: Namespace = {
|
||||
name: 'F95zone',
|
||||
url: 'f95zone.to',
|
||||
description: 'F95zone is a community for adult games and animations.',
|
||||
};
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
import { load } from 'cheerio';
|
||||
|
||||
import { config } from '@/config';
|
||||
import type { Route } from '@/types';
|
||||
import ofetch from '@/utils/ofetch';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
|
||||
import { processContent } from './utils';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/post/:thread/:postId',
|
||||
name: 'Post',
|
||||
maintainers: ['wsmbsbbz'],
|
||||
example: '/f95zone/post/vicineko-collection-2025-06-14-vicineko.84596/post-5909830',
|
||||
categories: ['game'],
|
||||
description: `Track content changes of a specific post. Uses the date \`[yyyy-mm-dd]\` in title for update detection.
|
||||
|
||||
URL format: \`https://f95zone.to/threads/{thread}/#post-{id}\` → replace \`#\` with \`/\` to get \`/f95zone/post/{thread}/post-{id}\`
|
||||
|
||||
Example: \`https://f95zone.to/threads/vicineko-collection-2025-06-14-vicineko.84596/#post-5909830\` → \`/f95zone/post/vicineko-collection-2025-06-14-vicineko.84596/post-5909830\`
|
||||
|
||||
Note: This route does not support Radar auto-detection because the post ID is in the URL hash (after \`#\`), which cannot be extracted by Radar. You need to manually construct the subscription URL.`,
|
||||
parameters: {
|
||||
thread: 'Thread slug with ID',
|
||||
postId: 'Post ID with `post-` prefix, replace `#` with `/` from browser URL',
|
||||
},
|
||||
features: {
|
||||
requireConfig: [
|
||||
{
|
||||
name: 'F95ZONE_COOKIE',
|
||||
optional: true,
|
||||
description: 'F95zone cookie for accessing restricted content.',
|
||||
},
|
||||
],
|
||||
requirePuppeteer: false,
|
||||
antiCrawler: false,
|
||||
supportBT: false,
|
||||
supportPodcast: false,
|
||||
supportScihub: false,
|
||||
nsfw: true,
|
||||
},
|
||||
radar: [],
|
||||
handler: async (ctx) => {
|
||||
const { thread, postId } = ctx.req.param();
|
||||
const baseUrl = 'https://f95zone.to';
|
||||
const link = `${baseUrl}/threads/${thread}/#${postId}`;
|
||||
|
||||
const response = await ofetch(link, {
|
||||
headers: {
|
||||
referer: baseUrl,
|
||||
...(config.f95zone.cookie ? { cookie: config.f95zone.cookie } : {}),
|
||||
},
|
||||
});
|
||||
|
||||
const $ = load(response);
|
||||
const title = $('h1.p-title-value').text().trim();
|
||||
const post = $(`article[data-content="${postId}"]`);
|
||||
const content = post.find('.bbWrapper').html() || '';
|
||||
const author = post.attr('data-author') || '';
|
||||
const postDate = post.find('time.u-dt').first().attr('datetime');
|
||||
const tags = $('a.tagItem')
|
||||
.toArray()
|
||||
.map((el) => $(el).text().trim());
|
||||
|
||||
// Extract [yyyy-mm-dd] from title for update tracking
|
||||
const dateMatch = title.match(/\[(\d{4}-\d{2}-\d{2})\]/);
|
||||
const updateDate = dateMatch?.[1];
|
||||
|
||||
return {
|
||||
title: `[F95zone] ${title}`,
|
||||
link,
|
||||
item: [
|
||||
{
|
||||
title: `[Updated] ${title}`,
|
||||
link,
|
||||
guid: updateDate ? `${link}_${updateDate}` : link,
|
||||
description: processContent(content),
|
||||
pubDate: updateDate ? parseDate(updateDate) : postDate ? parseDate(postDate) : undefined,
|
||||
author,
|
||||
category: tags,
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
};
|
||||
|
|
@ -0,0 +1,107 @@
|
|||
import { load } from 'cheerio';
|
||||
|
||||
import { config } from '@/config';
|
||||
import type { DataItem, Route } from '@/types';
|
||||
import ofetch from '@/utils/ofetch';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
|
||||
import { processContent } from './utils';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/thread/:thread',
|
||||
name: 'Thread',
|
||||
maintainers: ['wsmbsbbz'],
|
||||
example: '/f95zone/thread/ubermation-collection-2026-01-19-uebermation-uebermation.231247',
|
||||
categories: ['game'],
|
||||
description: `Track replies in a thread. Fetches the first page and the last page.
|
||||
|
||||
URL format: \`https://f95zone.to/threads/{thread}/\` → use \`{thread}\` as the parameter.
|
||||
|
||||
Example: \`https://f95zone.to/threads/ubermation-collection-2026-01-19-uebermation-uebermation.231247/\` → \`/f95zone/thread/ubermation-collection-2026-01-19-uebermation-uebermation.231247\`
|
||||
|
||||
Note: If you want to track a specific post's content changes (e.g., first post with download links), use the \`/f95zone/post\` route instead.`,
|
||||
parameters: {
|
||||
thread: 'Thread slug with ID, copy from browser URL after `/threads/`',
|
||||
},
|
||||
features: {
|
||||
requireConfig: [
|
||||
{
|
||||
name: 'F95ZONE_COOKIE',
|
||||
optional: true,
|
||||
description: 'F95zone cookie for accessing restricted content.',
|
||||
},
|
||||
],
|
||||
requirePuppeteer: false,
|
||||
antiCrawler: false,
|
||||
supportBT: false,
|
||||
supportPodcast: false,
|
||||
supportScihub: false,
|
||||
nsfw: true,
|
||||
},
|
||||
radar: [
|
||||
{
|
||||
source: ['f95zone.to/threads/:thread/*'],
|
||||
target: '/thread/:thread',
|
||||
},
|
||||
],
|
||||
handler: async (ctx) => {
|
||||
const { thread } = ctx.req.param();
|
||||
const baseUrl = 'https://f95zone.to';
|
||||
const threadLink = `${baseUrl}/threads/${thread}/`;
|
||||
|
||||
const headers = {
|
||||
referer: baseUrl,
|
||||
...(config.f95zone.cookie ? { cookie: config.f95zone.cookie } : {}),
|
||||
};
|
||||
|
||||
const firstPageResponse = await ofetch(threadLink, { headers });
|
||||
const $firstPage = load(firstPageResponse);
|
||||
const title = $firstPage('h1.p-title-value').text().trim();
|
||||
|
||||
const lastPageLink = $firstPage('ul.pageNav-main li.pageNav-page:last-child a').attr('href');
|
||||
const totalPages = lastPageLink ? Number.parseInt(lastPageLink.match(/page-(\d+)/)?.[1] || '1', 10) : 1;
|
||||
|
||||
const extractPosts = ($: ReturnType<typeof load>): DataItem[] =>
|
||||
$('article.message')
|
||||
.toArray()
|
||||
.flatMap((article) => {
|
||||
const $article = $(article);
|
||||
const postId = $article.attr('data-content')?.replace('post-', '');
|
||||
if (!postId) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const author = $article.find('.message-name a').text().trim();
|
||||
const postDate = $article.find('time.u-dt').attr('datetime');
|
||||
const content = $article.find('.bbWrapper').html() || '';
|
||||
const postLink = `${threadLink}post-${postId}`;
|
||||
|
||||
// Get post number from the attribution list (e.g., "#717")
|
||||
const postNumber = $article.find('.message-attribution-opposite--list li:last-child a').text().trim().replace('#', '') || postId;
|
||||
|
||||
return {
|
||||
title: `#${postNumber} by ${author}`,
|
||||
link: postLink,
|
||||
guid: postLink,
|
||||
description: processContent(content),
|
||||
pubDate: postDate ? parseDate(postDate) : undefined,
|
||||
author,
|
||||
};
|
||||
});
|
||||
|
||||
// Extract posts from the first page
|
||||
const allPosts: DataItem[] = [...extractPosts($firstPage)];
|
||||
|
||||
// Fetch the last page if there are multiple pages
|
||||
if (totalPages > 1) {
|
||||
const lastPageResponse = await ofetch(`${threadLink}page-${totalPages}`, { headers });
|
||||
allPosts.push(...extractPosts(load(lastPageResponse)));
|
||||
}
|
||||
|
||||
return {
|
||||
title: `[F95zone] ${title}`,
|
||||
link: threadLink,
|
||||
item: allPosts,
|
||||
};
|
||||
},
|
||||
};
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
import { load } from 'cheerio';
|
||||
|
||||
const ALLOWED_TAGS = new Set(['div', 'span', 'p', 'br', 'b', 'strong', 'i', 'em', 'u', 's', 'a', 'img', 'ul', 'ol', 'li', 'blockquote', 'hr', 'pre', 'code']);
|
||||
const ALLOWED_ATTRS: Record<string, string[]> = {
|
||||
a: ['href', 'target', 'rel'],
|
||||
img: ['src', 'alt', 'title', 'style'],
|
||||
div: ['style'],
|
||||
span: ['style'],
|
||||
};
|
||||
|
||||
export const processContent = (html: string): string => {
|
||||
const $ = load(html);
|
||||
|
||||
// Process images: use original URLs, remove duplicates
|
||||
const seenImages = new Set<string>();
|
||||
$('img').each((_, el) => {
|
||||
const $img = $(el);
|
||||
const $parent = $img.parent('a');
|
||||
let src = $parent.attr('href') || $img.attr('data-src') || $img.attr('src') || '';
|
||||
src = src.replace('/thumb/', '/');
|
||||
|
||||
if (!src || src.startsWith('data:') || seenImages.has(src)) {
|
||||
$img.remove();
|
||||
} else {
|
||||
seenImages.add(src);
|
||||
$img.attr('src', src).removeAttr('data-src');
|
||||
if ($parent.length) {
|
||||
$parent.replaceWith($img);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Remove unwanted tags completely
|
||||
$('button, script, style, noscript').remove();
|
||||
|
||||
// Remove disallowed tags but keep content
|
||||
let changed = true;
|
||||
while (changed) {
|
||||
changed = false;
|
||||
$('*').each((_, el) => {
|
||||
if (el.type === 'tag' && !ALLOWED_TAGS.has(el.name)) {
|
||||
$(el).replaceWith($(el).html() || '');
|
||||
changed = true;
|
||||
return false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Clean attributes
|
||||
$('*').each((_, el) => {
|
||||
if (el.type !== 'tag') {
|
||||
return;
|
||||
}
|
||||
const allowed = new Set(ALLOWED_ATTRS[el.name] || []);
|
||||
for (const attr of Object.keys(el.attribs || {})) {
|
||||
if (!allowed.has(attr)) {
|
||||
$(el).removeAttr(attr);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Remove empty divs
|
||||
$('div')
|
||||
.filter((_, el) => !$(el).html()?.trim())
|
||||
.remove();
|
||||
|
||||
return $.html() || '';
|
||||
};
|
||||
Loading…
Reference in New Issue