fix(route/nature): fix item.category (#13839)

* fix(route/nature): fix item.category * fix: shared cookieJar
2023-11-20 16:04:58 +00:00 · 2023-11-20 16:04:58 +00:00 · c8bb9dcc4a
parent bdc4b2bfbf
commit c8bb9dcc4a
7 changed files with 169 additions and 60 deletions
--- a/lib/v2/nature/cover.js
+++ b/lib/v2/nature/cover.js
@ -18,56 +18,46 @@ const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
 const { baseUrl, journalMap } = require('./utils');
+const { CookieJar } = require('tough-cookie');

 module.exports = async (ctx) => {
-    const journals = journalMap.items;
+    const cookieJar = new CookieJar();
+
+    await got('https://www.nature.com', { cookieJar });
+
+    const journals = journalMap.items
+        .filter((j) => j.id)
+        .map((j) => ({
+            ...j,
+            link: `${baseUrl}/${j.title}/current-issue`,
+        }));
+
    const out = await Promise.all(
-        journals
-            .filter((j) => j.id)
-            .map(async (journal) => {
-                // get the lastest volumn and issue id
-                const pageURL = `${baseUrl}/${journal.title}/current-issue`;
-                const cookieData = await got
-                    .extend({
-                        prefixUrl: 'https://idp.nature.com/authorize',
-                        followRedirect: false,
-                    })
-                    .get(`?response_type=cookie&client_id=grover&redirect_uri=${encodeURI(pageURL)}`)
-                    .then((response) => response.headers['set-cookie'].join(' '));
-                const issueURL = await got
-                    .extend({
-                        prefixUrl: pageURL,
-                        headers: {
-                            cookie: cookieData,
-                        },
-                        followRedirect: false,
-                    })
-                    .get('')
-                    .then((response) => response.headers.location);
-                const capturingRegex = /volumes\/(?<volumes>\d+)\/issues\/(?<issues>\d+)/;
-                const { volumes, issues } = issueURL.match(capturingRegex).groups;
+        journals.map((journal) =>
+            ctx.cache.tryGet(journal.link, async () => {
+                const { id } = journal;

-                const address = `${baseUrl}${issueURL}`;
-                return ctx.cache.tryGet(address, async () => {
-                    const id = journal.id;
-                    const imageURL = `https://media.springernature.com/full/springer-static/cover-hires/journal/${id}/${volumes}/${issues}?as=webp`;
-                    const contents = `<div align="center"><img src="${imageURL}" alt="Volume ${volumes} Issue ${issues}"></div>`;
+                const response = await got(journal.link, { cookieJar });

-                    const response = await got(address);
-                    const $ = cheerio.load(response.data);
-                    const date = $('title').text().split(',')[1].trim();
-                    const issueDescription = $('div[data-test=issue-description]').html() ?? '';
+                const capturingRegex = /volumes\/(?<volume>\d+)\/issues\/(?<issue>\d+)/;
+                const { volume, issue } = response.url.match(capturingRegex).groups;
+                const imageUrl = `https://media.springernature.com/full/springer-static/cover-hires/journal/${id}/${volume}/${issue}?as=webp`;
+                const contents = `<div align="center"><img src="${imageUrl}" alt="Volume ${volume} Issue ${issue}"></div>`;

-                    const single = {
-                        title: `${journal.name} | Volume ${volumes} Issue ${issues}`,
-                        description: contents + issueDescription,
-                        link: address,
-                        pubDate: parseDate(date, 'MMMM YYYY'),
-                    };
-                    return single;
-                });
+                const $ = cheerio.load(response.data);
+                const date = $('title').text().split(',')[1].trim();
+                const issueDescription = $('div[data-test=issue-description]').html() ?? '';
+
+                return {
+                    title: `${journal.name} | Volume ${volume} Issue ${issue}`,
+                    description: contents + issueDescription,
+                    link: response.url,
+                    pubDate: parseDate(date, 'MMMM YYYY'),
+                };
            })
+        )
    );
+
    ctx.state.data = {
        title: 'Nature Covers Story',
        description: 'Find out the cover story of some Nature journals.',
--- a/lib/v2/nature/highlight.js
+++ b/lib/v2/nature/highlight.js
@ -1,12 +1,12 @@
 const cheerio = require('cheerio');
 const got = require('@/utils/got');
-const { baseUrl, getArticleList, getArticle } = require('./utils');
+const { baseUrl, cookieJar, getArticleList, getArticle } = require('./utils');

 module.exports = async (ctx) => {
    const { journal = 'nature' } = ctx.params;
    const url = `${baseUrl}/${journal}/articles?type=research-highlight`;

-    const res = await got(url);
+    const res = await got(url, { cookieJar });
    const $ = cheerio.load(res.data);

    let items = getArticleList($);
--- a/lib/v2/nature/news-and-comment.js
+++ b/lib/v2/nature/news-and-comment.js
@ -14,13 +14,13 @@

 const cheerio = require('cheerio');
 const got = require('@/utils/got');
-const { baseUrl, getArticleList, getArticle } = require('./utils');
+const { baseUrl, cookieJar, getArticleList, getArticle } = require('./utils');

 module.exports = async (ctx) => {
    const journal = ctx.params.journal;
    const pageURL = `${baseUrl}/${journal}/news-and-comment`;

-    const pageResponse = await got(pageURL);
+    const pageResponse = await got(pageURL, { cookieJar });
    const pageCapture = cheerio.load(pageResponse.data);
    const pageDescription = pageCapture('meta[name=description]').attr('content') || 'Nature, a nature research journal';

--- a/lib/v2/nature/news.js
+++ b/lib/v2/nature/news.js
@ -1,11 +1,11 @@
 const cheerio = require('cheerio');
 const got = require('@/utils/got');
 const { parseDate } = require('@/utils/parse-date');
-const { baseUrl, getArticle } = require('./utils');
+const { baseUrl, cookieJar, getArticle } = require('./utils');

 module.exports = async (ctx) => {
    const url = `${baseUrl}/latest-news`;
-    const res = await got(url);
+    const res = await got(url, { cookieJar });
    const $ = cheerio.load(res.data);

    let items = $('.c-article-item__content')
--- a/lib/v2/nature/research.js
+++ b/lib/v2/nature/research.js
@ -15,13 +15,13 @@

 const cheerio = require('cheerio');
 const got = require('@/utils/got');
-const { baseUrl, getArticleList, getDataLayer, getArticle } = require('./utils');
+const { baseUrl, cookieJar, getArticleList, getDataLayer, getArticle } = require('./utils');

 module.exports = async (ctx) => {
    const journal = ctx.params.journal ?? 'nature';
    const pageURL = `${baseUrl}/${journal}/research-articles`;

-    const pageResponse = await got(pageURL);
+    const pageResponse = await got(pageURL, { cookieJar });
    const pageCapture = cheerio.load(pageResponse.data);

    const pageTitle = getDataLayer(pageCapture).content.journal.title;
--- a/lib/v2/nature/siteindex.js
+++ b/lib/v2/nature/siteindex.js
@ -1,9 +1,9 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const { baseUrl } = require('./utils');
+const { baseUrl, cookieJar } = require('./utils');

 module.exports = async (ctx) => {
-    const response = await got(`${baseUrl}/siteindex`);
+    const response = await got(`${baseUrl}/siteindex`, { cookieJar });
    const $ = cheerio.load(response.data);

    let items = $('li[class^="grid mq640-grid-12"]')
@ -11,7 +11,7 @@ module.exports = async (ctx) => {
        .map((item) => {
            item = $(item);
            return {
-                title: item.find('a').attr('href').replace('/', ''),
+                title: item.find('a').attr('href').replaceAll('/', ''),
                name: item.find('a').text(),
                link: baseUrl + item.find('a').attr('href'),
            };
@ -20,7 +20,7 @@ module.exports = async (ctx) => {
    items = await Promise.all(
        items.map((item) =>
            ctx.cache.tryGet(`nature:siteindex:${item.title}`, async () => {
-                const response = await got(item.link);
+                const response = await got(item.link, { cookieJar });
                const $ = cheerio.load(response.data);

                delete item.link;
@ -39,7 +39,7 @@ module.exports = async (ctx) => {

    ctx.state.data = {
        title: 'Nature siteindex',
-        link: baseUrl + '/siteindex',
+        link: response.url,
        item: items,
    };
    ctx.state.json = {
--- a/lib/v2/nature/utils.js
+++ b/lib/v2/nature/utils.js
@ -1,7 +1,7 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
-
+const { CookieJar } = require('tough-cookie');
 const baseUrl = 'https://www.nature.com';

 const fixFigure = (html) => {
@ -51,7 +51,9 @@ const getArticleList = (html) =>

 const getArticle = (item, ctx) =>
    ctx.cache.tryGet(item.link, async () => {
-        const response = await got(item.link);
+        const response = await got(item.link, {
+            cookieJar,
+        });
        const $ = cheerio.load(response.data);
        const responseUrl = new URL(response.url);

@ -59,7 +61,7 @@ const getArticle = (item, ctx) =>
            const meta = getDataLayer($);
            item.doi = meta.content.article?.doi;
            item.author = meta.content.contentInfo.authors.join(', ');
-            item.pubDate = parseDate(meta.content.contentInfo.publishedAt * 1000) || item.pubDate;
+            item.pubDate = parseDate(meta.content.contentInfo.publishedAt, 'X') || item.pubDate;
        } else {
            const meta = JSON.parse($('script[type="application/ld+json"]').html());
            const freeAccess = meta.mainEntity.isAccessibleForFree;
@ -69,7 +71,7 @@ const getArticle = (item, ctx) =>
                item.doi = meta.mainEntity.sameAs.replace('https://doi.org/', '');
            }
            item.author = meta.mainEntity.author.map((author) => author.name.replace(', ', ' ')).join(', ');
-            item.category = meta.mainEntity.keywords.split(',');
+            item.category = meta.mainEntity.keywords;
            item.pubDate = parseDate(meta.mainEntity.datePublished) || item.pubDate;

            fixFigure($);
@ -101,10 +103,12 @@ const getArticle = (item, ctx) =>
 const getDataLayer = (html) =>
    JSON.parse(
        html('script[data-test=dataLayer]')
-            .html()
+            .text()
            .match(/window\.dataLayer = \[(.*)\];/s)[1]
    );

+const cookieJar = new CookieJar();
+
 /**
 * This is generated by /nature/siteindex.debug.json
 */
@ -144,6 +148,10 @@ const journalMap = {
            id: '43747',
            description: '43747',
        },
+        {
+            title: 'bjcreports',
+            name: 'BJC Reports',
+        },
        {
            title: 'bcj',
            name: 'Blood Cancer Journal',
@ -234,6 +242,10 @@ const journalMap = {
            title: 'commsphys',
            name: 'Communications Physics',
        },
+        {
+            title: 'commspsychol',
+            name: 'Communications Psychology',
+        },
        {
            title: 'ejcn',
            name: 'European Journal of Clinical Nutrition',
@ -470,12 +482,20 @@ const journalMap = {
            id: '41589',
            description: '41589',
        },
+        {
+            title: 'natchemeng',
+            name: 'Nature Chemical Engineering',
+        },
        {
            title: 'nchem',
            name: 'Nature Chemistry',
            id: '41557',
            description: '41557',
        },
+        {
+            title: 'natcities',
+            name: 'Nature Cities',
+        },
        {
            title: 'nclimate',
            name: 'Nature Climate Change',
@ -573,6 +593,8 @@ const journalMap = {
        {
            title: 'natmentalhealth',
            name: 'Nature Mental Health',
+            id: '44220',
+            description: '44220',
        },
        {
            title: 'natmetab',
@ -631,6 +653,8 @@ const journalMap = {
        {
            title: 'natrevbioeng',
            name: 'Nature Reviews Bioengineering',
+            id: '44222',
+            description: '44222',
        },
        {
            title: 'nrc',
@ -672,6 +696,10 @@ const journalMap = {
            id: '43017',
            description: '43017',
        },
+        {
+            title: 'natrevelectreng',
+            name: 'Nature Reviews Electrical Engineering',
+        },
        {
            title: 'nrendo',
            name: 'Nature Reviews Endocrinology',
@ -781,6 +809,8 @@ const journalMap = {
        {
            title: 'natwater',
            name: 'Nature Water',
+            id: '44221',
+            description: '44221',
        },
        {
            title: 'npp',
@ -796,10 +826,18 @@ const journalMap = {
            title: 'npj2dmaterials',
            name: 'npj 2D Materials and Applications',
        },
+        {
+            title: 'npjadvmanuf',
+            name: 'npj Advanced Manufacturing',
+        },
        {
            title: 'npjamd',
            name: 'npj Aging',
        },
+        {
+            title: 'npjamar',
+            name: 'npj Antimicrobials and Resistance',
+        },
        {
            title: 'npjbiodivers',
            name: 'npj Biodiversity',
@ -808,18 +846,42 @@ const journalMap = {
            title: 'npjbiofilms',
            name: 'npj Biofilms and Microbiomes',
        },
+        {
+            title: 'npjbiolphysmech',
+            name: 'npj Biological Physics and Mechanics',
+        },
+        {
+            title: 'npjbts',
+            name: 'npj Biological Timing and Sleep',
+        },
+        {
+            title: 'npjbiosensing',
+            name: 'npj Biosensing',
+        },
        {
            title: 'npjbcancer',
            name: 'npj Breast Cancer',
        },
+        {
+            title: 'npjcardiohealth',
+            name: 'npj Cardiovascular Health',
+        },
        {
            title: 'npjcleanwater',
            name: 'npj Clean Water',
        },
+        {
+            title: 'npjclimataction',
+            name: 'npj Climate Action',
+        },
        {
            title: 'npjclimatsci',
            name: 'npj Climate and Atmospheric Science',
        },
+        {
+            title: 'npjcomplex',
+            name: 'npj Complexity',
+        },
        {
            title: 'npjcompumats',
            name: 'npj Computational Materials',
@ -836,18 +898,42 @@ const journalMap = {
            title: 'npjgenmed',
            name: 'npj Genomic Medicine',
        },
+        {
+            title: 'npjimaging',
+            name: 'npj Imaging',
+        },
        {
            title: 'npjmatdeg',
            name: 'npj Materials Degradation',
        },
+        {
+            title: 'npjmatsustain',
+            name: 'npj Materials Sustainability',
+        },
        {
            title: 'npjmentalhealth',
            name: 'npj Mental Health Research',
        },
+        {
+            title: 'npjmetabhealth',
+            name: 'npj Metabolic Health and Disease',
+        },
        {
            title: 'npjmgrav',
            name: 'npj Microgravity',
        },
+        {
+            title: 'npjnanophoton',
+            name: 'npj Nanophotonics',
+        },
+        {
+            title: 'npjnathazards',
+            name: 'npj Natural Hazards',
+        },
+        {
+            title: 'npjoceansustain',
+            name: 'npj Ocean Sustainability',
+        },
        {
            title: 'npjparkd',
            name: "npj Parkinson's Disease",
@ -872,6 +958,10 @@ const journalMap = {
            title: 'npjregenmed',
            name: 'npj Regenerative Medicine',
        },
+        {
+            title: 'npjrobot',
+            name: 'npj Robotics',
+        },
        {
            title: 'npjscifood',
            name: 'npj Science of Food',
@ -880,10 +970,26 @@ const journalMap = {
            title: 'npjscilearn',
            name: 'npj Science of Learning',
        },
+        {
+            title: 'npjspintronics',
+            name: 'npj Spintronics',
+        },
+        {
+            title: 'npjsustainagric',
+            name: 'npj Sustainable Agriculture',
+        },
+        {
+            title: 'npjsustainmobil',
+            name: 'npj Sustainable Mobility and Transport',
+        },
        {
            title: 'npjsba',
            name: 'npj Systems Biology and Applications',
        },
+        {
+            title: 'npjunconvcomput',
+            name: 'npj Unconventional Computing',
+        },
        {
            title: 'npjurbansustain',
            name: 'npj Urban Sustainability',
@ -892,6 +998,18 @@ const journalMap = {
            title: 'npjvaccines',
            name: 'npj Vaccines',
        },
+        {
+            title: 'npjviruses',
+            name: 'npj Viruses',
+        },
+        {
+            title: 'npjwomenshealth',
+            name: "npj Women's Health",
+        },
+        {
+            title: 'dpn',
+            name: 'NPP—Digital Psychiatry and Neuroscience',
+        },
        {
            title: 'nutd',
            name: 'Nutrition & Diabetes',
@ -973,6 +1091,7 @@ const journalMap = {

 module.exports = {
    baseUrl,
+    cookieJar,
    getArticle,
    getArticleList,
    getDataLayer,