102 lines
3.8 KiB
JavaScript
102 lines
3.8 KiB
JavaScript
const cheerio = require('cheerio');
|
|
const got = require('@/utils/got');
|
|
const { parseDate } = require('@/utils/parse-date');
|
|
const { art } = require('@/utils/render');
|
|
const path = require('path');
|
|
const { setCookies } = require('@/utils/puppeteer-utils');
|
|
const { CookieJar } = require('tough-cookie');
|
|
const logger = require('@/utils/logger');
|
|
|
|
module.exports = async (ctx) => {
|
|
const baseUrl = 'https://www.pnas.org';
|
|
const { topicPath } = ctx.params;
|
|
const link = `${baseUrl}/${topicPath ? topicPath : 'latest'}`;
|
|
|
|
let cookieJar = await ctx.cache.get('pnas:cookieJar');
|
|
const cacheMiss = !cookieJar;
|
|
cookieJar = cacheMiss ? new CookieJar() : CookieJar.fromJSON(cookieJar);
|
|
const { data: res } = await got(link, {
|
|
cookieJar,
|
|
});
|
|
if (cacheMiss) {
|
|
await ctx.cache.set('pnas:cookieJar', cookieJar.toJSON());
|
|
}
|
|
|
|
const $ = cheerio.load(res);
|
|
const list = $('.card--row-reversed .card-content')
|
|
.toArray()
|
|
.map((item) => {
|
|
item = $(item);
|
|
const a = item.find('.article-title a');
|
|
return {
|
|
title: a.text(),
|
|
link: new URL(a.attr('href'), baseUrl).href,
|
|
pubDate: parseDate(item.find('.card__meta__date').text()),
|
|
};
|
|
});
|
|
|
|
const browser = await require('@/utils/puppeteer')();
|
|
|
|
const out = await Promise.all(
|
|
list.map((item) =>
|
|
ctx.cache.tryGet(item.link, async () => {
|
|
const page = await browser.newPage();
|
|
await setCookies(page, await cookieJar.getCookieString(item.link), '.pnas.org');
|
|
await page.setRequestInterception(true);
|
|
page.on('request', (request) => {
|
|
request.resourceType() === 'document' ? request.continue() : request.abort();
|
|
});
|
|
logger.debug(`Requesting ${item.link}`);
|
|
await page.goto(item.link, {
|
|
waitUntil: 'domcontentloaded',
|
|
referer: link,
|
|
});
|
|
await page.waitForSelector('.core-container');
|
|
|
|
const res = await page.evaluate(() => document.documentElement.innerHTML);
|
|
await page.close();
|
|
|
|
const $ = cheerio.load(res);
|
|
const PNASdataLayer = JSON.parse(
|
|
$('script')
|
|
.text()
|
|
.match(/PNASdataLayer =(.*?);/)[1]
|
|
);
|
|
|
|
$('.signup-alert-ad, .citations-truncation button').remove();
|
|
|
|
const { keywords, topic } = PNASdataLayer.page.attributes;
|
|
|
|
item.category = [...keywords, topic];
|
|
item.author = PNASdataLayer.page.pageInfo.author;
|
|
item.doi = PNASdataLayer.page.pageInfo.DOI;
|
|
item.description = art(path.join(__dirname, 'templates', 'article.art'), {
|
|
access: PNASdataLayer.user.access === 'yes',
|
|
//
|
|
abstracts: $('#abstracts .core-container').html(),
|
|
//
|
|
articleBody: $('[property=articleBody]').html(),
|
|
//
|
|
dataAvailability: $('#data-availability').html(),
|
|
acknowledgments: $('#acknowledgments').html(),
|
|
supplementaryMaterials: $('#supplementary-materials').html(),
|
|
bibliography: $('#bibliography').html(),
|
|
});
|
|
|
|
return item;
|
|
})
|
|
)
|
|
);
|
|
|
|
browser.close();
|
|
|
|
ctx.state.data = {
|
|
title: `${$('.banner-widget__content h1').text()} - PNAS`,
|
|
description: $('.banner-widget__content p').text(),
|
|
image: 'https://www.pnas.org/favicon.ico',
|
|
language: 'en-US',
|
|
link,
|
|
item: out,
|
|
};
|
|
};
|