From f2ec5a8d86c87bd6a12aef97cf02c840ada322c4 Mon Sep 17 00:00:00 2001 From: Joshua Peek Date: Fri, 22 Sep 2023 11:54:30 -0700 Subject: [PATCH] fix(route): economist article body extraction (#13368) * fix(route): economist article body extraction * fix: limit items --------- --- lib/v2/economist/full.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/v2/economist/full.js b/lib/v2/economist/full.js index 4ad4262a8..936836060 100644 --- a/lib/v2/economist/full.js +++ b/lib/v2/economist/full.js @@ -9,7 +9,9 @@ const getArticleDetail = (link, ctx) => $('div.article-audio-player__center-tooltip').remove(); const nextData = JSON.parse($('head script[type="application/ld+json"]').first().text()); - const article = ($('figure[class^=css-]').first().parent().parent().html() || '') + $('p.article__body-text').parent().html(); + const figure = $('figure[class^=css-]').first().parent().parent().html() || ''; + const body = $('p[data-component="paragraph"]').parent().parent().html(); + const article = figure + body; const categories = nextData.keywords?.map((k) => k); return { article, categories }; @@ -20,7 +22,7 @@ module.exports = async (ctx) => { const feed = await parser.parseURL(`https://www.economist.com/${endpoint}/rss.xml`); const items = await Promise.all( - feed.items.map(async (item) => { + feed.items.slice(0, ctx.query.limit ? parseInt(ctx.query.limit, 10) : 30).map(async (item) => { const path = item.link.slice(item.link.lastIndexOf('/') + 1); const isNotCollection = !/^\d{4}-\d{2}-\d{2}$/.test(path); const itemDetails = isNotCollection ? await getArticleDetail(item.link, ctx) : null;