81 lines
3.0 KiB
JavaScript
81 lines
3.0 KiB
JavaScript
const cheerio = require('cheerio');
|
|
const { parseDate } = require('@/utils/parse-date');
|
|
const { getCookies, setCookies } = require('@/utils/puppeteer-utils');
|
|
const logger = require('@/utils/logger');
|
|
|
|
module.exports = async (ctx) => {
|
|
const { journal } = ctx.params;
|
|
const baseUrl = 'https://www.journals.uchicago.edu';
|
|
const link = `${baseUrl}/toc/${journal}/current`;
|
|
|
|
const browser = await require('@/utils/puppeteer')();
|
|
const page = await browser.newPage();
|
|
await page.setRequestInterception(true);
|
|
page.on('request', (request) => {
|
|
request.resourceType() === 'document' ? request.continue() : request.abort();
|
|
});
|
|
logger.debug(`Requesting ${link}`);
|
|
await page.goto(link, {
|
|
waitUntil: 'domcontentloaded',
|
|
});
|
|
const response = await page.evaluate(() => document.documentElement.innerHTML);
|
|
const cookies = await getCookies(page);
|
|
page.close();
|
|
const $ = cheerio.load(response);
|
|
|
|
const list = $('.issue-item__title')
|
|
.toArray()
|
|
.map((item) => ({ link: `${baseUrl}${$(item).find('a').attr('href')}` }));
|
|
|
|
const items = await Promise.all(
|
|
list.map((item) =>
|
|
ctx.cache.tryGet(item.link, async () => {
|
|
const page = await browser.newPage();
|
|
await setCookies(page, cookies, 'journals.uchicago.edu');
|
|
await page.setRequestInterception(true);
|
|
page.on('request', (request) => {
|
|
request.resourceType() === 'document' ? request.continue() : request.abort();
|
|
});
|
|
logger.debug(`Requesting ${item.link}`);
|
|
await page.goto(item.link, {
|
|
waitUntil: 'domcontentloaded',
|
|
referer: link,
|
|
});
|
|
const response = await page.evaluate(() => document.documentElement.innerHTML);
|
|
page.close();
|
|
|
|
const $ = cheerio.load(response);
|
|
|
|
item.title = $('head title').text();
|
|
item.pubDate = parseDate($('head meta[name="dc.Date"]').attr('content'));
|
|
item.doi = $('head meta[scheme="doi"]').attr('content');
|
|
item.author = $('.author-name span')
|
|
.toArray()
|
|
.map((author) => $(author).text())
|
|
.join(', ');
|
|
|
|
$('.figure__image').each((_, elem) => {
|
|
if (elem.attribs['data-lg-src']) {
|
|
$(elem).attr('src', `${baseUrl}${elem.attribs['data-lg-src']}`);
|
|
delete elem.attribs['data-lg-src'];
|
|
}
|
|
});
|
|
|
|
item.description = $('.article__body').html();
|
|
|
|
return item;
|
|
})
|
|
)
|
|
);
|
|
|
|
browser.close();
|
|
|
|
ctx.state.data = {
|
|
title: $('head title').text(),
|
|
description: $('.jumbotron-journal-info').text(),
|
|
link,
|
|
image: $('head meta[property="og:image"]').attr('content'),
|
|
item: items,
|
|
};
|
|
};
|