From b3d8bc861d809f73bdc65b8f8752d4e0007bc273 Mon Sep 17 00:00:00 2001
From: Stephen Zhou <38493346+hyoban@users.noreply.github.com>
Date: Tue, 15 Oct 2024 00:37:05 +0800
Subject: [PATCH] feat: extract content (#17099)
* feat: extract content
* chore: update
* cache.tryGet
* fix: cache whole item
---------
---
lib/routes/rsshub/transform/html.ts | 186 ++++++++++++++++------------
1 file changed, 109 insertions(+), 77 deletions(-)
diff --git a/lib/routes/rsshub/transform/html.ts b/lib/routes/rsshub/transform/html.ts
index d63c73782..55b897cbe 100644
--- a/lib/routes/rsshub/transform/html.ts
+++ b/lib/routes/rsshub/transform/html.ts
@@ -1,8 +1,10 @@
-import { Route } from '@/types';
+import { DataItem, Route } from '@/types';
import got from '@/utils/got';
import { load } from 'cheerio';
import { config } from '@/config';
import ConfigNotFoundError from '@/errors/types/config-not-found';
+import sanitizeHtml from 'sanitize-html';
+import cache from '@/utils/cache';
export const route: Route = {
path: '/transform/html/:url/:routeParams',
@@ -23,24 +25,24 @@ export const route: Route = {
supportScihub: false,
},
name: 'Transformation - HTML',
- maintainers: ['ttttmr'],
- handler,
+ maintainers: ['ttttmr', 'hyoban'],
description: `Pass URL and transformation rules to convert HTML/JSON into RSS.
Specify options (in the format of query string) in parameter \`routeParams\` parameter to extract data from HTML.
-| Key | Meaning | Accepted Values | Default |
-| ----------------- | -------------------------------------------------------------- | --------------- | ---------------------- |
-| \`title\` | The title of the RSS | \`string\` | Extract from \`
\` |
-| \`item\` | The HTML elements as \`item\` using CSS selector | \`string\` | html |
-| \`itemTitle\` | The HTML elements as \`title\` in \`item\` using CSS selector | \`string\` | \`item\` element |
-| \`itemTitleAttr\` | The attributes of \`title\` element as title | \`string\` | Element text |
-| \`itemLink\` | The HTML elements as \`link\` in \`item\` using CSS selector | \`string\` | \`item\` element |
-| \`itemLinkAttr\` | The attributes of \`link\` element as link | \`string\` | \`href\` |
-| \`itemDesc\` | The HTML elements as \`descrption\` in \`item\` using CSS selector | \`string\` | \`item\` element |
-| \`itemDescAttr\` | The attributes of \`descrption\` element as description | \`string\` | Element html |
-| \`itemPubDate\` | The HTML elements as \`pubDate\` in \`item\` using CSS selector | \`string\` | \`item\` element |
-| \`itemPubDateAttr\` | The attributes of \`pubDate\` element as pubDate | \`string\` | Element html |
+| Key | Meaning | Accepted Values | Default |
+| ------------------- | ------------------------------------------------------------------------------------------------------------- | --------------- | ------------------------ |
+| \`title\` | The title of the RSS | \`string\` | Extract from \`\` |
+| \`item\` | The HTML elements as \`item\` using CSS selector | \`string\` | html |
+| \`itemTitle\` | The HTML elements as \`title\` in \`item\` using CSS selector | \`string\` | \`item\` element |
+| \`itemTitleAttr\` | The attributes of \`title\` element as title | \`string\` | Element text |
+| \`itemLink\` | The HTML elements as \`link\` in \`item\` using CSS selector | \`string\` | \`item\` element |
+| \`itemLinkAttr\` | The attributes of \`link\` element as link | \`string\` | \`href\` |
+| \`itemDesc\` | The HTML elements as \`descrption\` in \`item\` using CSS selector | \`string\` | \`item\` element |
+| \`itemDescAttr\` | The attributes of \`descrption\` element as description | \`string\` | Element html |
+| \`itemPubDate\` | The HTML elements as \`pubDate\` in \`item\` using CSS selector | \`string\` | \`item\` element |
+| \`itemPubDateAttr\` | The attributes of \`pubDate\` element as pubDate | \`string\` | Element html |
+| \`itemContent\` | The HTML elements as \`description\` in \`item\` using CSS selector ( in \`itemLink\` page for full content ) | \`string\` | |
Parameters parsing in the above example:
@@ -54,66 +56,96 @@ Specify options (in the format of query string) in parameter \`routeParams\` par
| Parameter | Value |
| --------- | ------------------------------- |
| \`item\` | \`div[class='post-content'] p a\` |`,
+ handler: async (ctx) => {
+ if (!config.feature.allow_user_supply_unsafe_domain) {
+ throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
+ }
+ const url = ctx.req.param('url');
+ const response = await got({
+ method: 'get',
+ url,
+ });
+
+ const routeParams = new URLSearchParams(ctx.req.param('routeParams'));
+ const $ = load(response.data);
+ const rssTitle = routeParams.get('title') || $('title').text();
+ const item = routeParams.get('item') || 'html';
+ let items: DataItem[] = $(item)
+ .toArray()
+ .map((item) => {
+ try {
+ item = $(item);
+
+ const titleEle = routeParams.get('itemTitle') ? item.find(routeParams.get('itemTitle')) : item;
+ const title = routeParams.get('itemTitleAttr') ? titleEle.attr(routeParams.get('itemTitleAttr')) : titleEle.text();
+
+ let link;
+ const linkEle = routeParams.get('itemLink') ? item.find(routeParams.get('itemLink')) : item;
+ if (routeParams.get('itemLinkAttr')) {
+ link = linkEle.attr(routeParams.get('itemLinkAttr'));
+ } else {
+ link = linkEle.is('a') ? linkEle.attr('href') : linkEle.find('a').attr('href');
+ }
+ // 补全绝对链接
+ link = link.trim();
+ if (link && !link.startsWith('http')) {
+ link = `${new URL(url).origin}${link}`;
+ }
+
+ const descEle = routeParams.get('itemDesc') ? item.find(routeParams.get('itemDesc')) : item;
+ const desc = routeParams.get('itemDescAttr') ? descEle.attr(routeParams.get('itemDescAttr')) : descEle.html();
+
+ const pubDateEle = routeParams.get('itemPubDate') ? item.find(routeParams.get('itemPubDate')) : item;
+ const pubDate = routeParams.get('itemPubDateAttr') ? pubDateEle.attr(routeParams.get('itemPubDateAttr')) : pubDateEle.html();
+
+ return {
+ title,
+ link,
+ description: desc,
+ pubDate,
+ };
+ } catch {
+ return null;
+ }
+ })
+ .filter((i) => !!i);
+
+ const itemContentSelector = routeParams.get('itemContent');
+ if (itemContentSelector) {
+ items = await Promise.all(
+ items.map((item) => {
+ if (!item.link) {
+ return item;
+ }
+
+ return cache.tryGet(item.link, async () => {
+ const response = await got({
+ method: 'get',
+ url: item.link,
+ });
+ if (!response || typeof response === 'string') {
+ return item;
+ }
+
+ const $ = load(response.data);
+ const content = $(itemContentSelector).html();
+ if (!content) {
+ return item;
+ }
+
+ item.description = sanitizeHtml(content, { allowedTags: [...sanitizeHtml.defaults.allowedTags, 'img'] });
+
+ return item;
+ });
+ })
+ );
+ }
+
+ return {
+ title: rssTitle,
+ link: url,
+ description: `Proxy ${url}`,
+ item: items,
+ };
+ },
};
-
-async function handler(ctx) {
- if (!config.feature.allow_user_supply_unsafe_domain) {
- throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
- }
- const url = ctx.req.param('url');
- const response = await got({
- method: 'get',
- url,
- });
-
- const routeParams = new URLSearchParams(ctx.req.param('routeParams'));
- const $ = load(response.data);
- const rssTitle = routeParams.get('title') || $('title').text();
- const item = routeParams.get('item') || 'html';
- const items = $(item)
- .toArray()
- .map((item) => {
- try {
- item = $(item);
-
- const titleEle = routeParams.get('itemTitle') ? item.find(routeParams.get('itemTitle')) : item;
- const title = routeParams.get('itemTitleAttr') ? titleEle.attr(routeParams.get('itemTitleAttr')) : titleEle.text();
-
- let link;
- const linkEle = routeParams.get('itemLink') ? item.find(routeParams.get('itemLink')) : item;
- if (routeParams.get('itemLinkAttr')) {
- link = linkEle.attr(routeParams.get('itemLinkAttr'));
- } else {
- link = linkEle.is('a') ? linkEle.attr('href') : linkEle.find('a').attr('href');
- }
- // 补全绝对链接
- link = link.trim();
- if (link && !link.startsWith('http')) {
- link = `${new URL(url).origin}${link}`;
- }
-
- const descEle = routeParams.get('itemDesc') ? item.find(routeParams.get('itemDesc')) : item;
- const desc = routeParams.get('itemDescAttr') ? descEle.attr(routeParams.get('itemDescAttr')) : descEle.html();
-
- const pubDateEle = routeParams.get('itemPubDate') ? item.find(routeParams.get('itemPubDate')) : item;
- const pubDate = routeParams.get('itemPubDateAttr') ? pubDateEle.attr(routeParams.get('itemPubDateAttr')) : pubDateEle.html();
-
- return {
- title,
- link,
- description: desc,
- pubDate,
- };
- } catch {
- return null;
- }
- })
- .filter(Boolean);
-
- return {
- title: rssTitle,
- link: url,
- description: `Proxy ${url}`,
- item: items,
- };
-}