diff --git a/docs/en/new-media.md b/docs/en/new-media.md index 17a06ce77..869765981 100644 --- a/docs/en/new-media.md +++ b/docs/en/new-media.md @@ -772,6 +772,16 @@ Compared to the official one, this feed: +## TechPowerUp + +### Latest Content + + + +### Reviews + + + ## The Brain ### Blog diff --git a/lib/v2/techpowerup/index.js b/lib/v2/techpowerup/index.js new file mode 100644 index 000000000..a40c4f278 --- /dev/null +++ b/lib/v2/techpowerup/index.js @@ -0,0 +1,69 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const { parseDate } = require('@/utils/parse-date'); +const { baseUrl, headers, fixImages, parseReviews } = require('./utils'); + +module.exports = async (ctx) => { + const { data: response } = await got(baseUrl, { + headers, + }); + + const $ = cheerio.load(response); + + const list = $('.newspost') + .toArray() + .map((item) => { + item = $(item); + const a = item.find('h1 a'); + const date = item.find('time').attr('datetime'); + return { + title: a.text(), + link: baseUrl + a.attr('href'), + pubDate: date ? parseDate(date) : null, // 2023-05-21T16:05:14+00:00 + author: item.find('.byline address').text(), + category: item + .find('.byline .flags span') + .toArray() + .map((item) => $(item).text().trim()), + }; + }); + + const items = await Promise.all( + list.map((item) => + ctx.cache.tryGet(item.link, async () => { + const { data: response } = await got(item.link, { + headers, + }); + const $ = cheerio.load(response); + + fixImages($); + + if (item.link.includes('/review/')) { + await parseReviews($, item); + return item; + } + + // news + item.description = $('.newspost .text').html(); + item.category = [ + ...new Set([ + ...item.category, + ...$('.tags li a') + .toArray() + .map((item) => $(item).text()), + ]), + ]; + + return item; + }) + ) + ); + + ctx.state.data = { + title: 'TechPowerUp', + link: baseUrl, + language: 'en', + image: 'https://tpucdn.com/apple-touch-icon-v1684568903519.png', + item: items, + }; +}; diff --git a/lib/v2/techpowerup/maintainer.js b/lib/v2/techpowerup/maintainer.js new file mode 100644 index 000000000..388438c4c --- /dev/null +++ b/lib/v2/techpowerup/maintainer.js @@ -0,0 +1,5 @@ +module.exports = { + '': ['TonyRL'], + '/': ['TonyRL'], + '/review/:keyword?': ['TonyRL'], +}; diff --git a/lib/v2/techpowerup/radar.js b/lib/v2/techpowerup/radar.js new file mode 100644 index 000000000..b2719b08a --- /dev/null +++ b/lib/v2/techpowerup/radar.js @@ -0,0 +1,19 @@ +module.exports = { + 'techpowerup.com': { + _name: 'TechPowerUp', + '.': [ + { + title: 'Latest Content', + docs: 'https://docs.rsshub.app/en/new-media.html#techpowerup', + source: ['/'], + target: '/techpowerup', + }, + { + title: 'Reviews', + docs: 'https://docs.rsshub.app/en/new-media.html#techpowerup', + source: ['/review/search', '/review'], + target: (_, url) => `/techpowerup/review${new URL(url).searchParams.has('q') ? `/${new URL(url).searchParams.get('q')}` : ''}`, + }, + ], + }, +}; diff --git a/lib/v2/techpowerup/review.js b/lib/v2/techpowerup/review.js new file mode 100644 index 000000000..ff836a2c3 --- /dev/null +++ b/lib/v2/techpowerup/review.js @@ -0,0 +1,69 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const { parseDate } = require('@/utils/parse-date'); +const { baseUrl, headers, fixImages, parseReviews } = require('./utils'); + +module.exports = async (ctx) => { + const { keyword } = ctx.params; + + const url = new URL(`${baseUrl}/review/${keyword ? 'search/' : ''}`); + if (keyword) { + url.searchParams.set('q', keyword); + url.searchParams.set('_', Date.now()); + } + + const { data: response } = await got(url, { + headers, + }); + + const $ = cheerio.load(response); + + const list = $('.reviewlist-bit') + .toArray() + .map((item) => { + item = $(item); + const a = item.find('.title a'); + return { + title: a.text(), + link: baseUrl + a.attr('href'), + pubDate: parseDate(item.find('.date time').attr('datetime')), // 2023-05-21T16:05:14+00:00 + author: item + .find('.author') + .contents() + .filter((_, c) => c.type === 'text') + .text() + .trim(), + category: item + .find('.category') + .contents() + .filter((_, c) => c.type === 'text') + .text() + .trim(), + }; + }); + + const items = await Promise.all( + list.map((item) => + ctx.cache.tryGet(item.link, async () => { + const { data: response } = await got(item.link, { + headers, + }); + const $ = cheerio.load(response); + + fixImages($); + + await parseReviews($, item); + + return item; + }) + ) + ); + + ctx.state.data = { + title: 'Reviews | TechPowerUp', + link: url.href, + language: 'en', + image: 'https://tpucdn.com/apple-touch-icon-v1684568903519.png', + item: items, + }; +}; diff --git a/lib/v2/techpowerup/router.js b/lib/v2/techpowerup/router.js new file mode 100644 index 000000000..ca6dee4bb --- /dev/null +++ b/lib/v2/techpowerup/router.js @@ -0,0 +1,4 @@ +module.exports = (router) => { + router.get('/', require('./index')); + router.get('/review/:keyword?', require('./review')); +}; diff --git a/lib/v2/techpowerup/utils.js b/lib/v2/techpowerup/utils.js new file mode 100644 index 000000000..a0fca8d80 --- /dev/null +++ b/lib/v2/techpowerup/utils.js @@ -0,0 +1,84 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const { parseDate } = require('@/utils/parse-date'); +const md5 = require('@/utils/md5'); + +const baseUrl = 'https://www.techpowerup.com'; + +const headers = { + cookie: `botcheck=${md5(Date.now().toString())}`, +}; + +const fixImages = ($) => { + $('div.responsive-image-xx').each((_, d) => { + removeResponsiveStyle(d); + }); + $('figure').each((_, f) => { + removeFigureStyle(f); + }); + $('.newspost img').each((_, img) => { + hdImage(img); + }); +}; + +const hdImage = (img) => { + img.attribs.src = img.attribs.src.replace('_thm', '').replace('_small', ''); + if (img.parentNode.name === 'a' && img.parentNode.attribs['data-width'] && img.parentNode.attribs['data-height']) { + img.attribs.width = img.parentNode.attribs['data-width']; + img.attribs.height = img.parentNode.attribs['data-height']; + } +}; +const removeFigureStyle = (f) => { + delete f.attribs.style; +}; + +const removeResponsiveStyle = (div) => { + delete div.attribs.style; +}; + +const parseReviews = async ($, item) => { + const { review } = JSON.parse($('script[type="application/ld+json"]').text()); + + const content = $('.text'); + const nextPages = $('#pagesel option') + .toArray() + .map((a) => `${baseUrl}${a.attribs.value}`) + .slice(1, -1); + + if (nextPages.length) { + const pages = await Promise.all( + nextPages.map(async (url) => { + const { data: response } = await got(url, { + headers, + }); + const $ = cheerio.load(response); + $('.text div.responsive-image-xx').each((_, d) => { + removeResponsiveStyle(d); + }); + $('.text figure').each((_, f) => { + removeFigureStyle(f); + }); + $('.text img').each((_, img) => { + hdImage(img); + }); + return $('.text').html(); + }) + ); + content.append(pages); + } + + item.author = review.author.name; + item.pubDate = parseDate(review.datePublished); + item.updated = parseDate(review.dateModified); + item.description = content.html(); +}; + +module.exports = { + baseUrl, + headers, + fixImages, + hdImage, + parseReviews, + removeFigureStyle, + removeResponsiveStyle, +};