From 51ac5f87e84d7f2f637711f51a99de0802236d10 Mon Sep 17 00:00:00 2001 From: Tony Date: Tue, 31 Oct 2023 19:51:14 +0000 Subject: [PATCH] fix(route): scmp full text (#13667) * fix(route): scmp full text * fix: null node check --- lib/v2/scmp/index.js | 146 ++++++++++++++++++------------------------- lib/v2/scmp/utils.js | 60 ++++++++++++++++++ 2 files changed, 121 insertions(+), 85 deletions(-) create mode 100644 lib/v2/scmp/utils.js diff --git a/lib/v2/scmp/index.js b/lib/v2/scmp/index.js index 574acea8b..bf86b25a1 100644 --- a/lib/v2/scmp/index.js +++ b/lib/v2/scmp/index.js @@ -1,114 +1,90 @@ -const parser = require('@/utils/rss-parser'); const cheerio = require('cheerio'); const got = require('@/utils/got'); const { parseDate } = require('@/utils/parse-date'); -const chromeMobileUserAgent = require('@/utils/rand-user-agent')({ browser: 'chrome', os: 'android', device: 'mobile' }); +const { renderHTML } = require('./utils'); module.exports = async (ctx) => { const categoryId = ctx.params.category_id; const rssUrl = `https://www.scmp.com/rss/${categoryId}/feed`; - const feed = await parser.parseURL(rssUrl); + const { data: response } = await got(rssUrl); + const $ = cheerio.load(response, { + xmlMode: true, + }); + + const list = $('item') + .toArray() + .map((item) => { + item = $(item); + const enclosure = item.find('enclosure').first(); + const mediaContent = item.find('media\\:content').toArray()[0]; + const thumbnail = item.find('media\\:thumbnail').toArray()[0]; + return { + title: item.find('title').text(), + description: item.find('description').text(), + link: item.find('link').text().split('?utm_source')[0], + author: item.find('author').text(), + pubDate: parseDate(item.find('pubDate').text()), + enclosure_url: enclosure?.attr('url'), + enclosure_length: enclosure?.attr('length'), + enclosure_type: enclosure?.attr('type'), + media: { + content: Object.keys(mediaContent.attribs).reduce((data, key) => { + data[key] = mediaContent.attribs[key]; + return data; + }, {}), + thumbnail: thumbnail?.attribs + ? Object.keys(thumbnail.attribs).reduce((data, attr) => { + data[attr] = thumbnail.attribs[attr]; + return data; + }, {}) + : undefined, + }, + }; + }); const items = await Promise.all( - feed.items.map((item) => + list.map((item) => ctx.cache.tryGet(item.link, async () => { - // Fetch the AMP version - const url = item.link.replace(/^https:\/\/www\.scmp\.com/, 'https://amp.scmp.com'); - const response = await got(url, { - headers: { - 'User-Agent': chromeMobileUserAgent, - }, - }); - const html = response.data; - const $ = cheerio.load(html); - const content = $('div.article-body.clearfix'); + const { data: response, url } = await got(item.link); - // Cover - const cover = $('.article-images > amp-carousel > .i-amphtml-slides-container >.i-amphtml-slide-item > amp-img > img'); - - if (cover.length > 0) { - $(``).insertBefore(content[0].childNodes[0]); - $(cover).remove(); + if (new URL(url).hostname !== 'www.scmp.com') { + // e.g., https://multimedia.scmp.com/ + return item; } - // Summary - const summary = $('div.article-header__subhead > ul'); + const $ = cheerio.load(response); - // Metadata (categories & updatedAt) - const updatedAt = $('meta[itemprop="dateModified"]').attr('content'); - const publishedAt = item.pubDate || $('meta[itemprop="datePublished"]').attr('content'); + const nextData = JSON.parse($('script#__NEXT_DATA__').text()); + const { article } = nextData.props.pageProps.payload.data; - const categories = $('meta[name="keywords"]') - .attr('content') - .split(',') - .map((c) => c.trim()); + // item.nextData = article; - // Images - content.find('amp-img').each((i, e) => { - const img = $(`${e.attribs.alt}`); + item.summary = renderHTML(article.summary.json); + item.description = renderHTML(article.subHeadline.json) + renderHTML(article.images.find((i) => i.type === 'leading')) + renderHTML(article.body.json); + item.updated = parseDate(article.updatedDate, 'x'); + item.category = [...new Set([...article.topics.map((t) => t.name), ...article.sections.flatMap((t) => t.value.map((v) => v.name)), ...article.keywords.map((k) => k?.split(', '))])]; - // Caption follows, no need to handle caption - $(img).insertBefore(e); - $(e).remove(); - }); + // N.B. gallery in article is not rendered + // e.g., { type: 'div', attribs: { class: 'scmp-photo-gallery', 'data-gallery-nid': '3239409' }} + // from https://www.scmp.com/news/china/politics/article/3239355/li-keqiang-former-premier-china-dead - // iframes (youtube videos and interactive elements) - content.find('amp-iframe').each((i, e) => { - if ($(e).find('iframe').length > 0) { - const iframe = $(e).find('iframe')[0]; - $(iframe).insertBefore(e); - $(e).remove(); - } - }); - - content.find('div.video-wrapper > amp-iframe').each((i, e) => { - const iframe = $(``; + case 'leading': + case 'img': + return `
`${key}="${node.attribs[key]}"`) + .join(' ') + : `url="${node.url}"` // for leading + }>
${node.attribs?.title ?? node.title}
`; + case 'em': + case 'h3': + case 'li': + case 'ol': + case 'ul': + case 'p': + case 'strong': + case 'u': + return `<${node.type}>${renderHTML(node.children)}`; + case 'text': + return node.data; + case 'script': + case 'inline-ad-slot': + case 'inline-widget': + case 'track-viewed-percentage': + return ''; + default: + return `Unhandled type: ${node.type} ${JSON.stringify(node)}`; + } +}; + +module.exports = { + renderHTML, +};