diff --git a/docs/traditional-media.md b/docs/traditional-media.md index 17bb39843..79c8b7052 100644 --- a/docs/traditional-media.md +++ b/docs/traditional-media.md @@ -2071,6 +2071,10 @@ category 对应的关键词有 +### 作者 + + + ## 卫报 The Guardian 通过提取文章全文，以提供比官方源更佳的阅读体验。 diff --git a/lib/v2/cw/author.js b/lib/v2/cw/author.js new file mode 100644 index 000000000..85fe02f16 --- /dev/null +++ b/lib/v2/cw/author.js @@ -0,0 +1,18 @@ +const { baseUrl, parsePage } = require('./utils'); + +module.exports = async (ctx) => { + const browser = await require('@/utils/puppeteer')(); + + const { $, items } = await parsePage('author', browser, ctx); + + await browser.close(); + + ctx.state.data = { + title: $('head title').text(), + description: $('.authorTxt').text(), + link: `${baseUrl}/author/${ctx.params.channel}`, + image: $('.authorPhoto img').attr('src') || `${baseUrl}/assets_new/img/fbshare.jpg'`, + language: $('meta[property="og:locale"]').attr('content'), + item: items, + }; +}; diff --git a/lib/v2/cw/maintainer.js b/lib/v2/cw/maintainer.js index 51a7e83af..7aaacc960 100644 --- a/lib/v2/cw/maintainer.js +++ b/lib/v2/cw/maintainer.js @@ -1,4 +1,5 @@ module.exports = { + '/author/:channel': ['TonyRL'], '/master/:channel': ['TonyRL'], '/sub/:channel': ['TonyRL'], '/today': ['TonyRL'], diff --git a/lib/v2/cw/master.js b/lib/v2/cw/master.js index aa34918c1..9fe3b1c00 100644 --- a/lib/v2/cw/master.js +++ b/lib/v2/cw/master.js @@ -1,34 +1,16 @@ -const cheerio = require('cheerio'); -const { baseUrl, parseList, parseItems, getCookie, setCookies } = require('./utils'); +const { baseUrl, parsePage } = require('./utils'); module.exports = async (ctx) => { - const { channel } = ctx.params; - const pageUrl = `${baseUrl}/masterChannel.action`; - const browser = await require('@/utils/puppeteer')(); - const cookie = await getCookie(browser, ctx.cache.tryGet); - const page = await browser.newPage(); - await page.setRequestInterception(true); - page.on('request', (request) => { - request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort(); - }); - await setCookies(page, cookie, 'cw.com.tw'); - await page.goto(`${pageUrl}?idMasterChannel=${channel}`, { - waitUntil: 'domcontentloaded', - }); - const response = await page.evaluate(() => document.documentElement.innerHTML); - await page.close(); - const $ = cheerio.load(response); - - const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : 12); - const items = await parseItems(list, browser, ctx.cache.tryGet); + const { $, items } = await parsePage('master', browser, ctx); await browser.close(); ctx.state.data = { title: $('head title').text(), description: $('meta[name=description]').attr('content'), + link: `${baseUrl}/masterChannel.action?idMasterChannel=${ctx.params.channel}`, image: `${baseUrl}/assets_new/img/fbshare.jpg`, language: $('meta[property="og:locale"]').attr('content'), item: items, diff --git a/lib/v2/cw/radar.js b/lib/v2/cw/radar.js index 9fa3a27e4..c1869514e 100644 --- a/lib/v2/cw/radar.js +++ b/lib/v2/cw/radar.js @@ -20,6 +20,12 @@ module.exports = { source: ['/subchannel.action'], target: (_, url) => `/cw/sub/${new URL(url).searchParams.get('idSubChannel')}`, }, + { + title: '作者', + docs: 'https://docs.rsshub.app/traditional-media.html#tian-xia-za-zhi', + source: ['/author/:channel'], + target: '/cw/author/:channel', + }, ], }, }; diff --git a/lib/v2/cw/router.js b/lib/v2/cw/router.js index 6d7edc690..7d3536b6c 100644 --- a/lib/v2/cw/router.js +++ b/lib/v2/cw/router.js @@ -1,4 +1,5 @@ module.exports = (router) => { + router.get('/author/:channel', require('./author')); router.get('/master/:channel', require('./master')); router.get('/sub/:channel', require('./sub')); router.get('/today', require('./today')); diff --git a/lib/v2/cw/sub.js b/lib/v2/cw/sub.js index 134ff6166..768624324 100644 --- a/lib/v2/cw/sub.js +++ b/lib/v2/cw/sub.js @@ -1,34 +1,16 @@ -const cheerio = require('cheerio'); -const { baseUrl, parseList, parseItems, getCookie, setCookies } = require('./utils'); +const { baseUrl, parsePage } = require('./utils'); module.exports = async (ctx) => { - const { channel } = ctx.params; - const pageUrl = `${baseUrl}/subchannel.action`; - const browser = await require('@/utils/puppeteer')(); - const cookie = await getCookie(browser, ctx.cache.tryGet); - const page = await browser.newPage(); - await page.setRequestInterception(true); - page.on('request', (request) => { - request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort(); - }); - await setCookies(page, cookie, 'cw.com.tw'); - await page.goto(`${pageUrl}?idSubChannel=${channel}`, { - waitUntil: 'domcontentloaded', - }); - const response = await page.evaluate(() => document.documentElement.innerHTML); - await page.close(); - const $ = cheerio.load(response); - - const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : 12); - const items = await parseItems(list, browser, ctx.cache.tryGet); + const { $, items } = await parsePage('sub', browser, ctx); await browser.close(); ctx.state.data = { title: $('head title').text(), description: $('meta[name=description]').attr('content'), + link: `${baseUrl}/subchannel.action?idSubChannel=${ctx.params.channel}`, image: `${baseUrl}/assets_new/img/fbshare.jpg`, language: $('meta[property="og:locale"]').attr('content'), item: items, diff --git a/lib/v2/cw/today.js b/lib/v2/cw/today.js index d878b3245..7bd01787b 100644 --- a/lib/v2/cw/today.js +++ b/lib/v2/cw/today.js @@ -1,33 +1,16 @@ -const cheerio = require('cheerio'); -const { baseUrl, parseList, parseItems, getCookie, setCookies } = require('./utils'); +const { baseUrl, parsePage } = require('./utils'); module.exports = async (ctx) => { - const pageUrl = `${baseUrl}/today`; - const browser = await require('@/utils/puppeteer')(); - const cookie = await getCookie(browser, ctx.cache.tryGet); - const page = await browser.newPage(); - await page.setRequestInterception(true); - page.on('request', (request) => { - request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort(); - }); - await setCookies(page, cookie, 'cw.com.tw'); - await page.goto(pageUrl, { - waitUntil: 'domcontentloaded', - }); - const response = await page.evaluate(() => document.documentElement.innerHTML); - await page.close(); - const $ = cheerio.load(response); - - const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : 30); - const items = await parseItems(list, browser, ctx.cache.tryGet); + const { $, items } = await parsePage('today', browser, ctx); await browser.close(); ctx.state.data = { title: $('head title').text(), description: $('meta[name=description]').attr('content'), + link: `${baseUrl}/today`, image: `${baseUrl}/assets_new/img/fbshare.jpg`, language: $('meta[property="og:locale"]').attr('content'), item: items, diff --git a/lib/v2/cw/utils.js b/lib/v2/cw/utils.js index a3735a61d..32ce93626 100644 --- a/lib/v2/cw/utils.js +++ b/lib/v2/cw/utils.js @@ -1,10 +1,30 @@ const cheerio = require('cheerio'); const { parseDate } = require('@/utils/parse-date'); const { getCookies, setCookies } = require('@/utils/puppeteer-utils'); +const logger = require('@/utils/logger'); let cookie; const baseUrl = 'https://www.cw.com.tw'; +const pathMap = { + today: { + pageUrl: () => '/today', + limit: 30, + }, + master: { + pageUrl: (channel) => `/masterChannel.action?idMasterChannel=${channel}`, + limit: 12, + }, + sub: { + pageUrl: (channel) => `/subchannel.action?idSubChannel=${channel}`, + limit: 12, + }, + author: { + pageUrl: (channel) => `/author/${channel}`, + limit: 10, + }, +}; + const getCookie = async (browser, tryGet) => { if (!cookie) { cookie = await tryGet('cw:cookie', async () => { @@ -13,6 +33,7 @@ const getCookie = async (browser, tryGet) => { page.on('request', (request) => { request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort(); }); + logger.debug(`Requesting ${baseUrl}/user/get/cookie-bar`); await page.goto(`${baseUrl}/user/get/cookie-bar`, { waitUntil: 'domcontentloaded', }); @@ -24,6 +45,31 @@ const getCookie = async (browser, tryGet) => { return cookie; }; +const parsePage = async (path, browser, ctx) => { + const pageUrl = `${baseUrl}${pathMap[path].pageUrl(ctx.params.channel)}`; + + const cookie = await getCookie(browser, ctx.cache.tryGet); + const page = await browser.newPage(); + await page.setRequestInterception(true); + page.on('request', (request) => { + request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort(); + }); + await setCookies(page, cookie, 'cw.com.tw'); + logger.debug(`Requesting ${pageUrl}`); + await page.goto(pageUrl, { + waitUntil: 'domcontentloaded', + }); + + const response = await page.evaluate(() => document.documentElement.innerHTML); + await page.close(); + const $ = cheerio.load(response); + + const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : pathMap[path].limit); + const items = await parseItems(list, browser, ctx.cache.tryGet); + + return { $, items }; +}; + const parseList = ($, limit) => $('.caption') .toArray() @@ -47,6 +93,7 @@ const parseItems = (list, browser, tryGet) => request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort(); }); await setCookies(page, cookie, 'cw.com.tw'); + logger.debug(`Requesting ${item.link}`); await page.goto(item.link, { waitUntil: 'domcontentloaded', }); @@ -77,8 +124,10 @@ const parseItems = (list, browser, tryGet) => module.exports = { baseUrl, + pathMap, getCookie, setCookies, + parsePage, parseList, parseItems, };