From b3d8bc861d809f73bdc65b8f8752d4e0007bc273 Mon Sep 17 00:00:00 2001 From: Stephen Zhou <38493346+hyoban@users.noreply.github.com> Date: Tue, 15 Oct 2024 00:37:05 +0800 Subject: [PATCH] feat: extract content (#17099) * feat: extract content * chore: update * cache.tryGet * fix: cache whole item --------- --- lib/routes/rsshub/transform/html.ts | 186 ++++++++++++++++------------ 1 file changed, 109 insertions(+), 77 deletions(-) diff --git a/lib/routes/rsshub/transform/html.ts b/lib/routes/rsshub/transform/html.ts index d63c73782..55b897cbe 100644 --- a/lib/routes/rsshub/transform/html.ts +++ b/lib/routes/rsshub/transform/html.ts @@ -1,8 +1,10 @@ -import { Route } from '@/types'; +import { DataItem, Route } from '@/types'; import got from '@/utils/got'; import { load } from 'cheerio'; import { config } from '@/config'; import ConfigNotFoundError from '@/errors/types/config-not-found'; +import sanitizeHtml from 'sanitize-html'; +import cache from '@/utils/cache'; export const route: Route = { path: '/transform/html/:url/:routeParams', @@ -23,24 +25,24 @@ export const route: Route = { supportScihub: false, }, name: 'Transformation - HTML', - maintainers: ['ttttmr'], - handler, + maintainers: ['ttttmr', 'hyoban'], description: `Pass URL and transformation rules to convert HTML/JSON into RSS. Specify options (in the format of query string) in parameter \`routeParams\` parameter to extract data from HTML. -| Key | Meaning | Accepted Values | Default | -| ----------------- | -------------------------------------------------------------- | --------------- | ---------------------- | -| \`title\` | The title of the RSS | \`string\` | Extract from \`\` | -| \`item\` | The HTML elements as \`item\` using CSS selector | \`string\` | html | -| \`itemTitle\` | The HTML elements as \`title\` in \`item\` using CSS selector | \`string\` | \`item\` element | -| \`itemTitleAttr\` | The attributes of \`title\` element as title | \`string\` | Element text | -| \`itemLink\` | The HTML elements as \`link\` in \`item\` using CSS selector | \`string\` | \`item\` element | -| \`itemLinkAttr\` | The attributes of \`link\` element as link | \`string\` | \`href\` | -| \`itemDesc\` | The HTML elements as \`descrption\` in \`item\` using CSS selector | \`string\` | \`item\` element | -| \`itemDescAttr\` | The attributes of \`descrption\` element as description | \`string\` | Element html | -| \`itemPubDate\` | The HTML elements as \`pubDate\` in \`item\` using CSS selector | \`string\` | \`item\` element | -| \`itemPubDateAttr\` | The attributes of \`pubDate\` element as pubDate | \`string\` | Element html | +| Key | Meaning | Accepted Values | Default | +| ------------------- | ------------------------------------------------------------------------------------------------------------- | --------------- | ------------------------ | +| \`title\` | The title of the RSS | \`string\` | Extract from \`<title>\` | +| \`item\` | The HTML elements as \`item\` using CSS selector | \`string\` | html | +| \`itemTitle\` | The HTML elements as \`title\` in \`item\` using CSS selector | \`string\` | \`item\` element | +| \`itemTitleAttr\` | The attributes of \`title\` element as title | \`string\` | Element text | +| \`itemLink\` | The HTML elements as \`link\` in \`item\` using CSS selector | \`string\` | \`item\` element | +| \`itemLinkAttr\` | The attributes of \`link\` element as link | \`string\` | \`href\` | +| \`itemDesc\` | The HTML elements as \`descrption\` in \`item\` using CSS selector | \`string\` | \`item\` element | +| \`itemDescAttr\` | The attributes of \`descrption\` element as description | \`string\` | Element html | +| \`itemPubDate\` | The HTML elements as \`pubDate\` in \`item\` using CSS selector | \`string\` | \`item\` element | +| \`itemPubDateAttr\` | The attributes of \`pubDate\` element as pubDate | \`string\` | Element html | +| \`itemContent\` | The HTML elements as \`description\` in \`item\` using CSS selector ( in \`itemLink\` page for full content ) | \`string\` | | Parameters parsing in the above example: @@ -54,66 +56,96 @@ Specify options (in the format of query string) in parameter \`routeParams\` par | Parameter | Value | | --------- | ------------------------------- | | \`item\` | \`div[class='post-content'] p a\` |`, + handler: async (ctx) => { + if (!config.feature.allow_user_supply_unsafe_domain) { + throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); + } + const url = ctx.req.param('url'); + const response = await got({ + method: 'get', + url, + }); + + const routeParams = new URLSearchParams(ctx.req.param('routeParams')); + const $ = load(response.data); + const rssTitle = routeParams.get('title') || $('title').text(); + const item = routeParams.get('item') || 'html'; + let items: DataItem[] = $(item) + .toArray() + .map((item) => { + try { + item = $(item); + + const titleEle = routeParams.get('itemTitle') ? item.find(routeParams.get('itemTitle')) : item; + const title = routeParams.get('itemTitleAttr') ? titleEle.attr(routeParams.get('itemTitleAttr')) : titleEle.text(); + + let link; + const linkEle = routeParams.get('itemLink') ? item.find(routeParams.get('itemLink')) : item; + if (routeParams.get('itemLinkAttr')) { + link = linkEle.attr(routeParams.get('itemLinkAttr')); + } else { + link = linkEle.is('a') ? linkEle.attr('href') : linkEle.find('a').attr('href'); + } + // 补全绝对链接 + link = link.trim(); + if (link && !link.startsWith('http')) { + link = `${new URL(url).origin}${link}`; + } + + const descEle = routeParams.get('itemDesc') ? item.find(routeParams.get('itemDesc')) : item; + const desc = routeParams.get('itemDescAttr') ? descEle.attr(routeParams.get('itemDescAttr')) : descEle.html(); + + const pubDateEle = routeParams.get('itemPubDate') ? item.find(routeParams.get('itemPubDate')) : item; + const pubDate = routeParams.get('itemPubDateAttr') ? pubDateEle.attr(routeParams.get('itemPubDateAttr')) : pubDateEle.html(); + + return { + title, + link, + description: desc, + pubDate, + }; + } catch { + return null; + } + }) + .filter((i) => !!i); + + const itemContentSelector = routeParams.get('itemContent'); + if (itemContentSelector) { + items = await Promise.all( + items.map((item) => { + if (!item.link) { + return item; + } + + return cache.tryGet(item.link, async () => { + const response = await got({ + method: 'get', + url: item.link, + }); + if (!response || typeof response === 'string') { + return item; + } + + const $ = load(response.data); + const content = $(itemContentSelector).html(); + if (!content) { + return item; + } + + item.description = sanitizeHtml(content, { allowedTags: [...sanitizeHtml.defaults.allowedTags, 'img'] }); + + return item; + }); + }) + ); + } + + return { + title: rssTitle, + link: url, + description: `Proxy ${url}`, + item: items, + }; + }, }; - -async function handler(ctx) { - if (!config.feature.allow_user_supply_unsafe_domain) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } - const url = ctx.req.param('url'); - const response = await got({ - method: 'get', - url, - }); - - const routeParams = new URLSearchParams(ctx.req.param('routeParams')); - const $ = load(response.data); - const rssTitle = routeParams.get('title') || $('title').text(); - const item = routeParams.get('item') || 'html'; - const items = $(item) - .toArray() - .map((item) => { - try { - item = $(item); - - const titleEle = routeParams.get('itemTitle') ? item.find(routeParams.get('itemTitle')) : item; - const title = routeParams.get('itemTitleAttr') ? titleEle.attr(routeParams.get('itemTitleAttr')) : titleEle.text(); - - let link; - const linkEle = routeParams.get('itemLink') ? item.find(routeParams.get('itemLink')) : item; - if (routeParams.get('itemLinkAttr')) { - link = linkEle.attr(routeParams.get('itemLinkAttr')); - } else { - link = linkEle.is('a') ? linkEle.attr('href') : linkEle.find('a').attr('href'); - } - // 补全绝对链接 - link = link.trim(); - if (link && !link.startsWith('http')) { - link = `${new URL(url).origin}${link}`; - } - - const descEle = routeParams.get('itemDesc') ? item.find(routeParams.get('itemDesc')) : item; - const desc = routeParams.get('itemDescAttr') ? descEle.attr(routeParams.get('itemDescAttr')) : descEle.html(); - - const pubDateEle = routeParams.get('itemPubDate') ? item.find(routeParams.get('itemPubDate')) : item; - const pubDate = routeParams.get('itemPubDateAttr') ? pubDateEle.attr(routeParams.get('itemPubDateAttr')) : pubDateEle.html(); - - return { - title, - link, - description: desc, - pubDate, - }; - } catch { - return null; - } - }) - .filter(Boolean); - - return { - title: rssTitle, - link: url, - description: `Proxy ${url}`, - item: items, - }; -}