From 277dae093d974769d758041e0187f0e1e012d4f0 Mon Sep 17 00:00:00 2001 From: Ethan Shen <42264778+nczitzk@users.noreply.github.com> Date: Tue, 26 Jul 2022 23:03:38 +0800 Subject: [PATCH] =?UTF-8?q?fix(route):=20=E8=89=B2=E8=8A=B1=E5=A0=82=20wit?= =?UTF-8?q?h=20duplicate=20magnets=20(#10327)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(route): 色花堂 with duplicate magnets * fix typo * refactor: use WHATWG URL --- docs/multimedia.md | 4 +- lib/router.js | 10 ++-- lib/{routes => v2}/dsndsht23/index.js | 86 +++++++++++++-------------- lib/v2/dsndsht23/maintainer.js | 7 +++ lib/v2/dsndsht23/radar.js | 19 ++++++ lib/v2/dsndsht23/router.js | 7 +++ 6 files changed, 80 insertions(+), 53 deletions(-) rename lib/{routes => v2}/dsndsht23/index.js (51%) create mode 100644 lib/v2/dsndsht23/maintainer.js create mode 100644 lib/v2/dsndsht23/radar.js create mode 100644 lib/v2/dsndsht23/router.js diff --git a/docs/multimedia.md b/docs/multimedia.md index 29595ac55..95a22b07e 100644 --- a/docs/multimedia.md +++ b/docs/multimedia.md @@ -1443,11 +1443,11 @@ JavDB 有多个备用域名，本路由默认使用永久域名 -## 色花堂中文论坛 +## 色花堂 ### 分区帖子 - + **原创 BT 电影** diff --git a/lib/router.js b/lib/router.js index 6fa07be24..f04d80ad7 100644 --- a/lib/router.js +++ b/lib/router.js @@ -1639,11 +1639,11 @@ router.get('/enclavebooks/user/:uid', lazyloadRouteHandler('./routes/enclavebook router.get('/enclavebooks/collection/:uid', lazyloadRouteHandler('./routes/enclavebooks/collection.js')); // 色花堂 -router.get('/dsndsht23/picture/:subforumid', lazyloadRouteHandler('./routes/dsndsht23/index')); -router.get('/dsndsht23/bt/:subforumid?', lazyloadRouteHandler('./routes/dsndsht23/index')); -router.get('/dsndsht23/:subforumid?/:type?', lazyloadRouteHandler('./routes/dsndsht23/index')); -router.get('/dsndsht23/:subforumid?', lazyloadRouteHandler('./routes/dsndsht23/index')); -router.get('/dsndsht23', lazyloadRouteHandler('./routes/dsndsht23/index')); +// router.get('/dsndsht23/picture/:subforumid', lazyloadRouteHandler('./routes/dsndsht23/index')); +// router.get('/dsndsht23/bt/:subforumid?', lazyloadRouteHandler('./routes/dsndsht23/index')); +// router.get('/dsndsht23/:subforumid?/:type?', lazyloadRouteHandler('./routes/dsndsht23/index')); +// router.get('/dsndsht23/:subforumid?', lazyloadRouteHandler('./routes/dsndsht23/index')); +// router.get('/dsndsht23', lazyloadRouteHandler('./routes/dsndsht23/index')); // 数英网最新文章 router.get('/digitaling/index', lazyloadRouteHandler('./routes/digitaling/index')); diff --git a/lib/routes/dsndsht23/index.js b/lib/v2/dsndsht23/index.js similarity index 51% rename from lib/routes/dsndsht23/index.js rename to lib/v2/dsndsht23/index.js index 0d68ba0bc..1aac5fe1f 100644 --- a/lib/routes/dsndsht23/index.js +++ b/lib/v2/dsndsht23/index.js @@ -1,6 +1,6 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); -const url = require('url'); +const { parseDate } = require('@/utils/parse-date'); const host = 'https://www.sehuatang.net/'; @@ -26,7 +26,7 @@ const forumIdMaps = { }; module.exports = async (ctx) => { - const subformName = ctx.params.subforumid || 'gqzwzm'; + const subformName = ctx.params.subforumid ?? 'gqzwzm'; const subformId = subformName in forumIdMaps ? forumIdMaps[subformName] : subformName; const typefilter = ctx.params.type ? `&filter=typeid&typeid=${ctx.params.type}` : ''; const link = `${host}forum.php?mod=forumdisplay&orderby=dateline&fid=${subformId}${typefilter}`; @@ -35,9 +35,7 @@ module.exports = async (ctx) => { 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', }; let c; - const response = await got({ - method: 'get', - url: link, + const response = await got(link, { headers, hooks: { beforeRedirect: [ @@ -57,65 +55,61 @@ module.exports = async (ctx) => { const $ = cheerio.load(response.data); const list = $('#threadlisttableid tbody[id^=normalthread]') - .slice(0, 25) + .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 25) .map(function () { const info = { title: '[' + $(this).find('th em a').text() + '] ' + $(this).find('a.xst').text(), link: $(this).find('a.xst').attr('href'), - date: $(this).find('td.by').find('em span span').attr('title'), + date: $(this).find('td.by').find('em span span').attr('title') || $(this).find('td.by').find('em span').first().text(), }; return info; }) .get(); const out = await Promise.all( - list.map(async (info) => { + list.map((info) => { const title = info.title; const date = info.date; const itemUrl = host + info.link; - const cache = await ctx.cache.get(itemUrl); - if (cache) { - return Promise.resolve(JSON.parse(cache)); - } + return ctx.cache.tryGet(itemUrl, async () => { + const response = await got(itemUrl, { + headers, + }); - const response = await got.get(itemUrl, { headers }); - - const $ = cheerio.load(response.data); - const postMessage = $("td[id^='postmessage']").slice(0, 1); - const images = $(postMessage).find('img'); - for (let k = 0; k < images.length; k++) { - if (!$(images[k]).attr('file') || $(images[k]).attr('file') === 'undefined') { - $(images[k]).replaceWith(''); - } else { - $(images[k]).replaceWith(``); + const $ = cheerio.load(response.data); + const postMessage = $("td[id^='postmessage']").slice(0, 1); + const images = $(postMessage).find('img'); + for (let k = 0; k < images.length; k++) { + if (!$(images[k]).attr('file') || $(images[k]).attr('file') === 'undefined') { + $(images[k]).replaceWith(''); + } else { + $(images[k]).replaceWith(``); + } } - } - const description = (postMessage.html() || '抓取原帖失败').replace(/ignore_js_op/g, 'div'); + const description = (postMessage.html() || '抓取原帖失败').replace(/ignore_js_op/g, 'div'); - const single = { - title, - link: itemUrl, - description, - pubDate: new Date(date).toUTCString(), - }; - const magnet = postMessage.find('div.blockcode li').text(); - const isMag = magnet.startsWith('magnet'); - const torrent = postMessage.find('a[href^=forum\\.php\\?mod\\=attachment]:not([href$=nothumb\\=yes])').attr('href'); - - const hasEnclosureUrl = isMag || torrent !== undefined; - if (hasEnclosureUrl) { - const enclosureUrl = isMag ? magnet : url.resolve(host, torrent); - const enclosure = { - enclosure_url: enclosureUrl, - enclosure_type: isMag ? 'application/x-bittorrent' : 'application/octet-stream', + const single = { + title, + link: itemUrl, + description, + pubDate: date ? parseDate(date) : null, }; - Object.assign(single, enclosure); - } - if (description !== '抓取原帖失败') { - ctx.cache.set(itemUrl, JSON.stringify(single)); - } - return Promise.resolve(single); + const magnet = postMessage.find('div.blockcode li').first().text(); + const isMag = magnet.startsWith('magnet'); + const torrent = postMessage.find('a[href^=forum\\.php\\?mod\\=attachment]:not([href$=nothumb\\=yes])').attr('href'); + + const hasEnclosureUrl = isMag || torrent !== undefined; + if (hasEnclosureUrl) { + const enclosureUrl = isMag ? magnet : new URL(torrent, host).href; + const enclosure = { + enclosure_url: enclosureUrl, + enclosure_type: isMag ? 'application/x-bittorrent' : 'application/octet-stream', + }; + Object.assign(single, enclosure); + } + return single; + }); }) ); diff --git a/lib/v2/dsndsht23/maintainer.js b/lib/v2/dsndsht23/maintainer.js new file mode 100644 index 000000000..599f29ca4 --- /dev/null +++ b/lib/v2/dsndsht23/maintainer.js @@ -0,0 +1,7 @@ +module.exports = { + '/bt/:subforumid?': ['qiwihui', 'junfengP', 'nczitzk'], + '/picture/:subforumid': ['qiwihui', 'junfengP', 'nczitzk'], + '/:subforumid?/:type?': ['qiwihui', 'junfengP', 'nczitzk'], + '/:subforumid?': ['qiwihui', 'junfengP', 'nczitzk'], + '/': ['qiwihui', 'junfengP', 'nczitzk'], +}; diff --git a/lib/v2/dsndsht23/radar.js b/lib/v2/dsndsht23/radar.js new file mode 100644 index 000000000..1037947f0 --- /dev/null +++ b/lib/v2/dsndsht23/radar.js @@ -0,0 +1,19 @@ +module.exports = { + 'sehuatang.net': { + _name: '色花堂', + '.': [ + { + title: '分区帖子', + docs: 'https://docs.rsshub.app/multimedia.html#se-hua-tang-fen-qu-tie-zi', + source: ['/:category', '/'], + target: (params, url) => { + const theUrl = new URL(url); + const matches = String(theUrl).match(/forum-(\d)+-\d+/); + const fid = theUrl.searchParams.get('fid') || (matches ? matches[1] : ''); + const tid = theUrl.searchParams.get('typeid'); + return `/dsndsht23${fid ? `/${fid}` : ''}${tid ? `/${tid}` : ''}`; + }, + }, + ], + }, +}; diff --git a/lib/v2/dsndsht23/router.js b/lib/v2/dsndsht23/router.js new file mode 100644 index 000000000..77b1145c7 --- /dev/null +++ b/lib/v2/dsndsht23/router.js @@ -0,0 +1,7 @@ +module.exports = function (router) { + router.get('/bt/:subforumid?', require('./index')); + router.get('/picture/:subforumid', require('./index')); + router.get('/:subforumid?/:type?', require('./index')); + router.get('/:subforumid?', require('./index')); + router.get('', require('./index')); +};