From 34783aa686d2647edc4d894ddac97725d6007006 Mon Sep 17 00:00:00 2001 From: DIYgod Date: Thu, 22 Feb 2024 02:03:07 +0800 Subject: [PATCH] feat: add rsshub routes --- lib/v3/rsshub/maintainer.js | 6 +++ lib/v3/rsshub/radar.js | 13 ++++++ lib/v3/rsshub/router.js | 6 +++ lib/v3/rsshub/routes.js | 65 ++++++++++++++++++++++++++++++ lib/v3/rsshub/transform/html.js | 65 ++++++++++++++++++++++++++++++ lib/v3/rsshub/transform/json.js | 58 ++++++++++++++++++++++++++ lib/v3/rsshub/transform/sitemap.js | 50 +++++++++++++++++++++++ 7 files changed, 263 insertions(+) create mode 100644 lib/v3/rsshub/maintainer.js create mode 100644 lib/v3/rsshub/radar.js create mode 100644 lib/v3/rsshub/router.js create mode 100644 lib/v3/rsshub/routes.js create mode 100644 lib/v3/rsshub/transform/html.js create mode 100644 lib/v3/rsshub/transform/json.js create mode 100644 lib/v3/rsshub/transform/sitemap.js diff --git a/lib/v3/rsshub/maintainer.js b/lib/v3/rsshub/maintainer.js new file mode 100644 index 000000000..158998b45 --- /dev/null +++ b/lib/v3/rsshub/maintainer.js @@ -0,0 +1,6 @@ +module.exports = { + '/routes/:lang?': ['DIYgod'], + '/transform/html/:url/:routeParams': ['ttttmr'], + '/transform/json/:url/:routeParams': ['ttttmr'], + '/transform/sitemap/:url/:routeParams?': ['flrngel'], +}; diff --git a/lib/v3/rsshub/radar.js b/lib/v3/rsshub/radar.js new file mode 100644 index 000000000..8327f7fec --- /dev/null +++ b/lib/v3/rsshub/radar.js @@ -0,0 +1,13 @@ +module.exports = { + 'rsshub.app': { + _name: 'RSSHub', + docs: [ + { + title: '有新路由啦', + docs: 'https://docs.rsshub.app/routes/program-update#rsshub', + source: ['/*'], + target: '/rsshub/routes', + }, + ], + }, +}; diff --git a/lib/v3/rsshub/router.js b/lib/v3/rsshub/router.js new file mode 100644 index 000000000..7e580dcc5 --- /dev/null +++ b/lib/v3/rsshub/router.js @@ -0,0 +1,6 @@ +module.exports = (router) => { + router.get('/routes/:lang?', require('./routes')); + router.get('/transform/html/:url/:routeParams', require('./transform/html')); + router.get('/transform/json/:url/:routeParams', require('./transform/json')); + router.get('/transform/sitemap/:url/:routeParams?', require('./transform/sitemap')); +}; diff --git a/lib/v3/rsshub/routes.js b/lib/v3/rsshub/routes.js new file mode 100644 index 000000000..9e9c2bc28 --- /dev/null +++ b/lib/v3/rsshub/routes.js @@ -0,0 +1,65 @@ +import got from '@/utils/got'; +const cheerio = require('cheerio'); + +module.exports = async (ctx) => { + const isEnglish = ctx.req.param('lang') !== 'zh'; + + const lang = isEnglish ? '' : 'zh/'; + const types = [ + 'social-media', + 'new-media', + 'traditional-media', + 'bbs', + 'blog', + 'programming', + 'design', + 'live', + 'multimedia', + 'picture', + 'anime', + 'program-update', + 'university', + 'forecast', + 'travel', + 'shopping', + 'game', + 'reading', + 'government', + 'study', + 'journal', + 'finance', + 'other', + ]; + const all = await Promise.all( + types.map(async (type) => { + const response = await got(`https://docs.rsshub.app/${lang}routes/${type}`); + + const data = response.data; + + const $ = cheerio.load(data); + const page = $('.page').toArray(); + const item = $('.routeBlock').toArray(); + return { page, item, type }; + }) + ); + const list = all.flatMap(({ page, item, type }) => item.map((item) => ({ page, item, type }))); + + ctx.set('data', { + title: isEnglish ? 'RSSHub has new routes' : 'RSSHub 有新路由啦', + link: 'https://docs.rsshub.app', + description: isEnglish ? 'Everything is RSSible' : '万物皆可 RSS', + language: isEnglish ? 'en-us' : 'zh-cn', + item: list.map(({ page, item, type }) => { + const $ = cheerio.load(page); + item = $(item); + const h2Title = item.prevAll('h2').eq(0); + const h3Title = item.prevAll('h3').eq(0); + return { + title: `${h2Title.text().trim()} - ${h3Title.text().trim()}`, + description: item.html(), + link: `https://docs.rsshub.app/${lang}routes/${type}#${encodeURIComponent(h2Title.find('.hash-link').attr('href') && h3Title.find('.hash-link').attr('href')?.substring(1))}`, + guid: item.attr('id'), + }; + }), + }); +}; diff --git a/lib/v3/rsshub/transform/html.js b/lib/v3/rsshub/transform/html.js new file mode 100644 index 000000000..0073580da --- /dev/null +++ b/lib/v3/rsshub/transform/html.js @@ -0,0 +1,65 @@ +import got from '@/utils/got'; +const cheerio = require('cheerio'); +import { config } from '@/config'; + +module.exports = async (ctx) => { + if (!config.feature.allow_user_supply_unsafe_domain) { + throw new Error(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); + } + const url = ctx.req.param('url'); + const response = await got({ + method: 'get', + url, + }); + + const routeParams = new URLSearchParams(ctx.req.param('routeParams')); + const $ = cheerio.load(response.data); + const rssTitle = routeParams.get('title') || $('title').text(); + const item = routeParams.get('item') || 'html'; + const items = $(item) + .toArray() + .map((item) => { + try { + item = $(item); + + const titleEle = routeParams.get('itemTitle') ? item.find(routeParams.get('itemTitle')) : item; + const title = routeParams.get('itemTitleAttr') ? titleEle.attr(routeParams.get('itemTitleAttr')) : titleEle.text(); + + let link; + const linkEle = routeParams.get('itemLink') ? item.find(routeParams.get('itemLink')) : item; + if (routeParams.get('itemLinkAttr')) { + link = linkEle.attr(routeParams.get('itemLinkAttr')); + } else { + link = linkEle.is('a') ? linkEle.attr('href') : linkEle.find('a').attr('href'); + } + // 补全绝对链接 + link = link.trim(); + if (link && !link.startsWith('http')) { + link = `${new URL(url).origin}${link}`; + } + + const descEle = routeParams.get('itemDesc') ? item.find(routeParams.get('itemDesc')) : item; + const desc = routeParams.get('itemDescAttr') ? descEle.attr(routeParams.get('itemDescAttr')) : descEle.html(); + + const pubDateEle = routeParams.get('itemPubDate') ? item.find(routeParams.get('itemPubDate')) : item; + const pubDate = routeParams.get('itemPubDateAttr') ? pubDateEle.attr(routeParams.get('itemPubDateAttr')) : pubDateEle.html(); + + return { + title, + link, + description: desc, + pubDate, + }; + } catch { + return null; + } + }) + .filter(Boolean); + + ctx.set('data', { + title: rssTitle, + link: url, + description: `Proxy ${url}`, + item: items, + }); +}; diff --git a/lib/v3/rsshub/transform/json.js b/lib/v3/rsshub/transform/json.js new file mode 100644 index 000000000..5ab74d2a3 --- /dev/null +++ b/lib/v3/rsshub/transform/json.js @@ -0,0 +1,58 @@ +import got from '@/utils/got'; +const cheerio = require('cheerio'); +import { config } from '@/config'; + +function jsonGet(obj, attr) { + if (typeof attr !== 'string') { + return obj; + } + // a.b.c + // a.b[0].c => a.b.0.c + for (const key of attr.split('.')) { + obj = obj[key]; + } + return obj; +} + +module.exports = async (ctx) => { + if (!config.feature.allow_user_supply_unsafe_domain) { + throw new Error(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); + } + const url = ctx.req.param('url'); + const response = await got({ + method: 'get', + url, + }); + + const routeParams = new URLSearchParams(ctx.req.param('routeParams')); + let rssTitle = routeParams.get('title'); + if (!rssTitle) { + const resp = await got({ + method: 'get', + url: new URL(url).origin, + }); + const $ = cheerio.load(resp.data); + rssTitle = $('title').text(); + } + + const items = jsonGet(response.data, routeParams.get('item')).map((item) => { + let link = jsonGet(item, routeParams.get('itemLink')).trim(); + // 补全绝对链接 + if (link && !link.startsWith('http')) { + link = `${new URL(url).origin}${link}`; + } + return { + title: jsonGet(item, routeParams.get('itemTitle')), + link, + description: routeParams.get('itemDesc') ? jsonGet(item, routeParams.get('itemDesc')) : '', + pubDate: routeParams.get('itemPubDate') ? jsonGet(item, routeParams.get('itemPubDate')) : '', + }; + }); + + ctx.set('data', { + title: rssTitle, + link: url, + description: `Proxy ${url}`, + item: items, + }); +}; diff --git a/lib/v3/rsshub/transform/sitemap.js b/lib/v3/rsshub/transform/sitemap.js new file mode 100644 index 000000000..4d7946e42 --- /dev/null +++ b/lib/v3/rsshub/transform/sitemap.js @@ -0,0 +1,50 @@ +import got from '@/utils/got'; +const cheerio = require('cheerio'); +import { config } from '@/config'; + +module.exports = async (ctx) => { + if (!config.feature.allow_user_supply_unsafe_domain) { + throw new Error(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); + } + const url = ctx.req.param('url'); + const response = await got({ + method: 'get', + url, + }); + + const routeParams = new URLSearchParams(ctx.req.param('routeParams')); + const $ = cheerio.load(response.data, { xmlMode: true }); + + const rssTitle = routeParams.get('title') || ($('urlset url').length && $('urlset url').first().find('loc').text() ? $('urlset url').first().find('loc').text() : 'Sitemap'); + + const urls = $('urlset url').toArray(); + const items = + urls && urls.length + ? urls + .map((item) => { + try { + const title = $(item).find('loc').text() || ''; + const link = $(item).find('loc').text() || ''; + const description = $(item).find('loc').text() || ''; + const pubDate = $(item).find('lastmod').text() || undefined; + + return { + title, + link, + description, + pubDate, + }; + } catch { + return null; + } + }) + .filter(Boolean) + : []; + + ctx.set('data', { + title: rssTitle, + link: url, + description: `Proxy ${url}`, + item: items, + }); +};