diff --git a/lib/v2/nature/cover.js b/lib/v2/nature/cover.js index c08bb161d..cecc930ea 100644 --- a/lib/v2/nature/cover.js +++ b/lib/v2/nature/cover.js @@ -18,56 +18,46 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); const { parseDate } = require('@/utils/parse-date'); const { baseUrl, journalMap } = require('./utils'); +const { CookieJar } = require('tough-cookie'); module.exports = async (ctx) => { - const journals = journalMap.items; + const cookieJar = new CookieJar(); + + await got('https://www.nature.com', { cookieJar }); + + const journals = journalMap.items + .filter((j) => j.id) + .map((j) => ({ + ...j, + link: `${baseUrl}/${j.title}/current-issue`, + })); + const out = await Promise.all( - journals - .filter((j) => j.id) - .map(async (journal) => { - // get the lastest volumn and issue id - const pageURL = `${baseUrl}/${journal.title}/current-issue`; - const cookieData = await got - .extend({ - prefixUrl: 'https://idp.nature.com/authorize', - followRedirect: false, - }) - .get(`?response_type=cookie&client_id=grover&redirect_uri=${encodeURI(pageURL)}`) - .then((response) => response.headers['set-cookie'].join(' ')); - const issueURL = await got - .extend({ - prefixUrl: pageURL, - headers: { - cookie: cookieData, - }, - followRedirect: false, - }) - .get('') - .then((response) => response.headers.location); - const capturingRegex = /volumes\/(?\d+)\/issues\/(?\d+)/; - const { volumes, issues } = issueURL.match(capturingRegex).groups; + journals.map((journal) => + ctx.cache.tryGet(journal.link, async () => { + const { id } = journal; - const address = `${baseUrl}${issueURL}`; - return ctx.cache.tryGet(address, async () => { - const id = journal.id; - const imageURL = `https://media.springernature.com/full/springer-static/cover-hires/journal/${id}/${volumes}/${issues}?as=webp`; - const contents = `
Volume ${volumes} Issue ${issues}
`; + const response = await got(journal.link, { cookieJar }); - const response = await got(address); - const $ = cheerio.load(response.data); - const date = $('title').text().split(',')[1].trim(); - const issueDescription = $('div[data-test=issue-description]').html() ?? ''; + const capturingRegex = /volumes\/(?\d+)\/issues\/(?\d+)/; + const { volume, issue } = response.url.match(capturingRegex).groups; + const imageUrl = `https://media.springernature.com/full/springer-static/cover-hires/journal/${id}/${volume}/${issue}?as=webp`; + const contents = `
Volume ${volume} Issue ${issue}
`; - const single = { - title: `${journal.name} | Volume ${volumes} Issue ${issues}`, - description: contents + issueDescription, - link: address, - pubDate: parseDate(date, 'MMMM YYYY'), - }; - return single; - }); + const $ = cheerio.load(response.data); + const date = $('title').text().split(',')[1].trim(); + const issueDescription = $('div[data-test=issue-description]').html() ?? ''; + + return { + title: `${journal.name} | Volume ${volume} Issue ${issue}`, + description: contents + issueDescription, + link: response.url, + pubDate: parseDate(date, 'MMMM YYYY'), + }; }) + ) ); + ctx.state.data = { title: 'Nature Covers Story', description: 'Find out the cover story of some Nature journals.', diff --git a/lib/v2/nature/highlight.js b/lib/v2/nature/highlight.js index f8a4bf0cd..d8c6a6a8b 100644 --- a/lib/v2/nature/highlight.js +++ b/lib/v2/nature/highlight.js @@ -1,12 +1,12 @@ const cheerio = require('cheerio'); const got = require('@/utils/got'); -const { baseUrl, getArticleList, getArticle } = require('./utils'); +const { baseUrl, cookieJar, getArticleList, getArticle } = require('./utils'); module.exports = async (ctx) => { const { journal = 'nature' } = ctx.params; const url = `${baseUrl}/${journal}/articles?type=research-highlight`; - const res = await got(url); + const res = await got(url, { cookieJar }); const $ = cheerio.load(res.data); let items = getArticleList($); diff --git a/lib/v2/nature/news-and-comment.js b/lib/v2/nature/news-and-comment.js index 9b0ebd046..d0c73e534 100644 --- a/lib/v2/nature/news-and-comment.js +++ b/lib/v2/nature/news-and-comment.js @@ -14,13 +14,13 @@ const cheerio = require('cheerio'); const got = require('@/utils/got'); -const { baseUrl, getArticleList, getArticle } = require('./utils'); +const { baseUrl, cookieJar, getArticleList, getArticle } = require('./utils'); module.exports = async (ctx) => { const journal = ctx.params.journal; const pageURL = `${baseUrl}/${journal}/news-and-comment`; - const pageResponse = await got(pageURL); + const pageResponse = await got(pageURL, { cookieJar }); const pageCapture = cheerio.load(pageResponse.data); const pageDescription = pageCapture('meta[name=description]').attr('content') || 'Nature, a nature research journal'; diff --git a/lib/v2/nature/news.js b/lib/v2/nature/news.js index 911edf0e6..717695fc5 100644 --- a/lib/v2/nature/news.js +++ b/lib/v2/nature/news.js @@ -1,11 +1,11 @@ const cheerio = require('cheerio'); const got = require('@/utils/got'); const { parseDate } = require('@/utils/parse-date'); -const { baseUrl, getArticle } = require('./utils'); +const { baseUrl, cookieJar, getArticle } = require('./utils'); module.exports = async (ctx) => { const url = `${baseUrl}/latest-news`; - const res = await got(url); + const res = await got(url, { cookieJar }); const $ = cheerio.load(res.data); let items = $('.c-article-item__content') diff --git a/lib/v2/nature/research.js b/lib/v2/nature/research.js index 3d683bb31..c28a0b108 100644 --- a/lib/v2/nature/research.js +++ b/lib/v2/nature/research.js @@ -15,13 +15,13 @@ const cheerio = require('cheerio'); const got = require('@/utils/got'); -const { baseUrl, getArticleList, getDataLayer, getArticle } = require('./utils'); +const { baseUrl, cookieJar, getArticleList, getDataLayer, getArticle } = require('./utils'); module.exports = async (ctx) => { const journal = ctx.params.journal ?? 'nature'; const pageURL = `${baseUrl}/${journal}/research-articles`; - const pageResponse = await got(pageURL); + const pageResponse = await got(pageURL, { cookieJar }); const pageCapture = cheerio.load(pageResponse.data); const pageTitle = getDataLayer(pageCapture).content.journal.title; diff --git a/lib/v2/nature/siteindex.js b/lib/v2/nature/siteindex.js index 055e67f0d..302affc88 100644 --- a/lib/v2/nature/siteindex.js +++ b/lib/v2/nature/siteindex.js @@ -1,9 +1,9 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); -const { baseUrl } = require('./utils'); +const { baseUrl, cookieJar } = require('./utils'); module.exports = async (ctx) => { - const response = await got(`${baseUrl}/siteindex`); + const response = await got(`${baseUrl}/siteindex`, { cookieJar }); const $ = cheerio.load(response.data); let items = $('li[class^="grid mq640-grid-12"]') @@ -11,7 +11,7 @@ module.exports = async (ctx) => { .map((item) => { item = $(item); return { - title: item.find('a').attr('href').replace('/', ''), + title: item.find('a').attr('href').replaceAll('/', ''), name: item.find('a').text(), link: baseUrl + item.find('a').attr('href'), }; @@ -20,7 +20,7 @@ module.exports = async (ctx) => { items = await Promise.all( items.map((item) => ctx.cache.tryGet(`nature:siteindex:${item.title}`, async () => { - const response = await got(item.link); + const response = await got(item.link, { cookieJar }); const $ = cheerio.load(response.data); delete item.link; @@ -39,7 +39,7 @@ module.exports = async (ctx) => { ctx.state.data = { title: 'Nature siteindex', - link: baseUrl + '/siteindex', + link: response.url, item: items, }; ctx.state.json = { diff --git a/lib/v2/nature/utils.js b/lib/v2/nature/utils.js index ce7dfe6f8..c3507ef48 100644 --- a/lib/v2/nature/utils.js +++ b/lib/v2/nature/utils.js @@ -1,7 +1,7 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); const { parseDate } = require('@/utils/parse-date'); - +const { CookieJar } = require('tough-cookie'); const baseUrl = 'https://www.nature.com'; const fixFigure = (html) => { @@ -51,7 +51,9 @@ const getArticleList = (html) => const getArticle = (item, ctx) => ctx.cache.tryGet(item.link, async () => { - const response = await got(item.link); + const response = await got(item.link, { + cookieJar, + }); const $ = cheerio.load(response.data); const responseUrl = new URL(response.url); @@ -59,7 +61,7 @@ const getArticle = (item, ctx) => const meta = getDataLayer($); item.doi = meta.content.article?.doi; item.author = meta.content.contentInfo.authors.join(', '); - item.pubDate = parseDate(meta.content.contentInfo.publishedAt * 1000) || item.pubDate; + item.pubDate = parseDate(meta.content.contentInfo.publishedAt, 'X') || item.pubDate; } else { const meta = JSON.parse($('script[type="application/ld+json"]').html()); const freeAccess = meta.mainEntity.isAccessibleForFree; @@ -69,7 +71,7 @@ const getArticle = (item, ctx) => item.doi = meta.mainEntity.sameAs.replace('https://doi.org/', ''); } item.author = meta.mainEntity.author.map((author) => author.name.replace(', ', ' ')).join(', '); - item.category = meta.mainEntity.keywords.split(','); + item.category = meta.mainEntity.keywords; item.pubDate = parseDate(meta.mainEntity.datePublished) || item.pubDate; fixFigure($); @@ -101,10 +103,12 @@ const getArticle = (item, ctx) => const getDataLayer = (html) => JSON.parse( html('script[data-test=dataLayer]') - .html() + .text() .match(/window\.dataLayer = \[(.*)\];/s)[1] ); +const cookieJar = new CookieJar(); + /** * This is generated by /nature/siteindex.debug.json */ @@ -144,6 +148,10 @@ const journalMap = { id: '43747', description: '43747', }, + { + title: 'bjcreports', + name: 'BJC Reports', + }, { title: 'bcj', name: 'Blood Cancer Journal', @@ -234,6 +242,10 @@ const journalMap = { title: 'commsphys', name: 'Communications Physics', }, + { + title: 'commspsychol', + name: 'Communications Psychology', + }, { title: 'ejcn', name: 'European Journal of Clinical Nutrition', @@ -470,12 +482,20 @@ const journalMap = { id: '41589', description: '41589', }, + { + title: 'natchemeng', + name: 'Nature Chemical Engineering', + }, { title: 'nchem', name: 'Nature Chemistry', id: '41557', description: '41557', }, + { + title: 'natcities', + name: 'Nature Cities', + }, { title: 'nclimate', name: 'Nature Climate Change', @@ -573,6 +593,8 @@ const journalMap = { { title: 'natmentalhealth', name: 'Nature Mental Health', + id: '44220', + description: '44220', }, { title: 'natmetab', @@ -631,6 +653,8 @@ const journalMap = { { title: 'natrevbioeng', name: 'Nature Reviews Bioengineering', + id: '44222', + description: '44222', }, { title: 'nrc', @@ -672,6 +696,10 @@ const journalMap = { id: '43017', description: '43017', }, + { + title: 'natrevelectreng', + name: 'Nature Reviews Electrical Engineering', + }, { title: 'nrendo', name: 'Nature Reviews Endocrinology', @@ -781,6 +809,8 @@ const journalMap = { { title: 'natwater', name: 'Nature Water', + id: '44221', + description: '44221', }, { title: 'npp', @@ -796,10 +826,18 @@ const journalMap = { title: 'npj2dmaterials', name: 'npj 2D Materials and Applications', }, + { + title: 'npjadvmanuf', + name: 'npj Advanced Manufacturing', + }, { title: 'npjamd', name: 'npj Aging', }, + { + title: 'npjamar', + name: 'npj Antimicrobials and Resistance', + }, { title: 'npjbiodivers', name: 'npj Biodiversity', @@ -808,18 +846,42 @@ const journalMap = { title: 'npjbiofilms', name: 'npj Biofilms and Microbiomes', }, + { + title: 'npjbiolphysmech', + name: 'npj Biological Physics and Mechanics', + }, + { + title: 'npjbts', + name: 'npj Biological Timing and Sleep', + }, + { + title: 'npjbiosensing', + name: 'npj Biosensing', + }, { title: 'npjbcancer', name: 'npj Breast Cancer', }, + { + title: 'npjcardiohealth', + name: 'npj Cardiovascular Health', + }, { title: 'npjcleanwater', name: 'npj Clean Water', }, + { + title: 'npjclimataction', + name: 'npj Climate Action', + }, { title: 'npjclimatsci', name: 'npj Climate and Atmospheric Science', }, + { + title: 'npjcomplex', + name: 'npj Complexity', + }, { title: 'npjcompumats', name: 'npj Computational Materials', @@ -836,18 +898,42 @@ const journalMap = { title: 'npjgenmed', name: 'npj Genomic Medicine', }, + { + title: 'npjimaging', + name: 'npj Imaging', + }, { title: 'npjmatdeg', name: 'npj Materials Degradation', }, + { + title: 'npjmatsustain', + name: 'npj Materials Sustainability', + }, { title: 'npjmentalhealth', name: 'npj Mental Health Research', }, + { + title: 'npjmetabhealth', + name: 'npj Metabolic Health and Disease', + }, { title: 'npjmgrav', name: 'npj Microgravity', }, + { + title: 'npjnanophoton', + name: 'npj Nanophotonics', + }, + { + title: 'npjnathazards', + name: 'npj Natural Hazards', + }, + { + title: 'npjoceansustain', + name: 'npj Ocean Sustainability', + }, { title: 'npjparkd', name: "npj Parkinson's Disease", @@ -872,6 +958,10 @@ const journalMap = { title: 'npjregenmed', name: 'npj Regenerative Medicine', }, + { + title: 'npjrobot', + name: 'npj Robotics', + }, { title: 'npjscifood', name: 'npj Science of Food', @@ -880,10 +970,26 @@ const journalMap = { title: 'npjscilearn', name: 'npj Science of Learning', }, + { + title: 'npjspintronics', + name: 'npj Spintronics', + }, + { + title: 'npjsustainagric', + name: 'npj Sustainable Agriculture', + }, + { + title: 'npjsustainmobil', + name: 'npj Sustainable Mobility and Transport', + }, { title: 'npjsba', name: 'npj Systems Biology and Applications', }, + { + title: 'npjunconvcomput', + name: 'npj Unconventional Computing', + }, { title: 'npjurbansustain', name: 'npj Urban Sustainability', @@ -892,6 +998,18 @@ const journalMap = { title: 'npjvaccines', name: 'npj Vaccines', }, + { + title: 'npjviruses', + name: 'npj Viruses', + }, + { + title: 'npjwomenshealth', + name: "npj Women's Health", + }, + { + title: 'dpn', + name: 'NPP—Digital Psychiatry and Neuroscience', + }, { title: 'nutd', name: 'Nutrition & Diabetes', @@ -973,6 +1091,7 @@ const journalMap = { module.exports = { baseUrl, + cookieJar, getArticle, getArticleList, getDataLayer,