From fe3416146c41f0726ecad64af015ca2ad895c876 Mon Sep 17 00:00:00 2001 From: Jiahao Lee Date: Wed, 16 Apr 2025 18:24:41 +0800 Subject: [PATCH] feat(route): add JPMC Institute (#18818) * feat(route): add JPMorgan Chase Institute * feat(route/jpmc): fix page content scraping * chore(route/jpmc): remove deprecated routes * fix(route/jpmc): make gh es linter happy * fix(route/jpmc): not caching the items array --- .../jpmorganchase/research.js | 52 --------- lib/routes/jpmorganchase/namespace.ts | 9 ++ lib/routes/jpmorganchase/research.ts | 107 ++++++++++++++++++ 3 files changed, 116 insertions(+), 52 deletions(-) delete mode 100644 lib/routes-deprecated/jpmorganchase/research.js create mode 100644 lib/routes/jpmorganchase/namespace.ts create mode 100644 lib/routes/jpmorganchase/research.ts diff --git a/lib/routes-deprecated/jpmorganchase/research.js b/lib/routes-deprecated/jpmorganchase/research.js deleted file mode 100644 index c2d27540b..000000000 --- a/lib/routes-deprecated/jpmorganchase/research.js +++ /dev/null @@ -1,52 +0,0 @@ -const got = require('@/utils/got'); -const cheerio = require('cheerio'); -const { parseDate } = require('@/utils/parse-date'); - -const base = 'https://institute.jpmorganchase.com'; -const url = `${base}/institute/research`; - -const parseDetails = (link, ctx) => { - const fullLink = `${base}${link}`; - return ctx.cache.tryGet(fullLink, async () => { - const response = await got({ - url: fullLink, - }); - const $ = cheerio.load(response.data); - const authors = []; - $('.author-name').each((i, elem) => { - authors.push($(elem).text()); - }); - - return { - category: $('.eyebrow').text(), - author: authors.filter(Boolean).join(', '), - title: $('title').text() + ' | ' + $('.copy-wrap p').text(), - description: $('.jpmc-wrapper').html(), - link: fullLink, - pubDate: parseDate($('.date-field').text(), 'MMMM YYYY'), - }; - }); -}; - -module.exports = async (ctx) => { - const response = await got({ - method: 'get', - url, - }); - - const title = 'All Reports'; - const $ = cheerio.load(response.data); - - const items = $('.item a') - .map((i, item) => { - const link = item.attribs.href; - return parseDetails(link, ctx); - }) - .get(); - ctx.state.data = { - title: `${title} - JPMorgan Chase Institute`, - link: url, - description: `${title} - JPMorgan Chase Institute`, - item: await Promise.all(items), - }; -}; diff --git a/lib/routes/jpmorganchase/namespace.ts b/lib/routes/jpmorganchase/namespace.ts new file mode 100644 index 000000000..2d6157f63 --- /dev/null +++ b/lib/routes/jpmorganchase/namespace.ts @@ -0,0 +1,9 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'JPMorgan Chase', + url: 'www.jpmorganchase.com', + zh: { + name: '摩根大通', + }, +}; diff --git a/lib/routes/jpmorganchase/research.ts b/lib/routes/jpmorganchase/research.ts new file mode 100644 index 000000000..19f93b329 --- /dev/null +++ b/lib/routes/jpmorganchase/research.ts @@ -0,0 +1,107 @@ +import { Data, DataItem, Route } from '@/types'; +import ofetch from '@/utils/ofetch'; +import cache from '@/utils/cache'; +import { load } from 'cheerio'; +import { parseDate } from '@/utils/parse-date'; + +const base = 'https://www.jpmorganchase.com'; +const frontPageUrl = `${base}/institute/all-topics`; +const indexUrl = `${base}/services/json/v1/dynamic-grid.service/parent=jpmorganchase/global/US/en/home/institute/all-topics&comp=root/content-parsys/dynamic_grid&page=p1.json`; + +export const route: Route = { + path: '/', + example: '/jpmorganchase', + categories: ['finance'], + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['jpmorganchase.com/institute/all-topics'], + target: '/', + }, + ], + name: 'Research Topics', + maintainers: ['dousha'], + handler, + url: 'www.jpmorganchase.com/institute/all-topics', +}; + +type PartitionMeta = { + 'total-items': number; + page: number; + 'page-size': number; + 'max-pages': number; + 'partition-size': string; +}; + +type IndexEntry = { + title: string; + date: string; + hideDate: boolean; + description: string; + type: string; + link: string; + linkText: string; + image: string; +}; + +async function fetchIndexEntires(): Promise { + const response = await ofetch(indexUrl); + if (!('meta' in response)) { + return []; + } + + const meta = response.meta as PartitionMeta; + const maxItemCount = Number(meta['partition-size']); + + return (response.items as IndexEntry[]).slice(0, maxItemCount); +} + +function fetchDataItem(entry: IndexEntry): Promise { + const url = `${base}${entry.link}`; + + return cache.tryGet(url, async () => { + let authors: string[] = []; + let description: string = ''; + let category: string[] = []; + let articleDate: string = entry.date; + const pageContent: string = await ofetch(url); + + if (pageContent.length > 0) { + const $ = load(pageContent); + + category = [$('.eyebrow').text()]; + authors = $('.author-name') + .toArray() + .map((el) => $(el).text().trim()); + articleDate = $('.date').text().trim() || entry.date; + description = $('.root').children('div').children('div:eq(1)').html() || ''; + } + + return { + category, + author: authors.join(', '), + title: entry.title, + description, + link: url, + pubDate: parseDate(articleDate), + } satisfies DataItem; + }) as Promise; +} + +async function handler(): Promise { + const entires = await fetchIndexEntires(); + const items = await Promise.all(entires.map((it) => fetchDataItem(it))); + + return { + title: 'All Topics - JPMorganChase Institute', + link: frontPageUrl, + item: items, + }; +}