fix(route): scitation - use Puppeteer due to the obstacle by cloudflare challenge (#10190)

* fix(route): scitation - use Puppeteer due to the obstacle by cloudflare challenge

* feat: update utils.js and journal.js

* feat: prepare for section.js

* feat: add section.js

* fix(section.js): fix "section" length undifine error

* feat: format

* update(docs): add puppeteer="1"

docs/en/journal.md

@@ -334,11 +334,11 @@ _only support Science Journal_
 
 ### Journal
 
-<RouteEn author="Derekmini auto-bot-ty" example="/scitation/aapt/ajp" path="/scitation/:pub/:jrn" :paramsDesc="['Publisher, the part of the URL before `scitation.org`','Journal, the part of the URL after `/toc/`']" radar="1" rssbud="1"/>
+<RouteEn author="Derekmini auto-bot-ty" example="/scitation/aapt/ajp" path="/scitation/:pub/:jrn" :paramsDesc="['Publisher, the part of the URL before `scitation.org`','Journal, the part of the URL after `/toc/`']" radar="1" rssbud="1" puppeteer="1"/>
 
 ### Section
 
-<RouteEn author="Derekmini auto-bot-ty" example="/scitation/aapt/ajp/COMPUTATIONAL+PHYSICS" path="/scitation/:pub/:jrn/:sec" :paramsDesc="['Publisher, the part of the URL before `scitation.org`','Journal, the part of the URL after `/toc/`','Section, the `tocSection` part of the URL']" radar="1" rssbud="1"/>
+<RouteEn author="Derekmini auto-bot-ty" example="/scitation/aapt/ajp/COMPUTATIONAL+PHYSICS" path="/scitation/:pub/:jrn/:sec" :paramsDesc="['Publisher, the part of the URL before `scitation.org`','Journal, the part of the URL after `/toc/`','Section, the `tocSection` part of the URL']" radar="1" rssbud="1" puppeteer="1"/>
 
 ## Springer
 

docs/journal.md

@@ -301,11 +301,11 @@ You can get all short name of a journal from <https://www.nature.com/siteindex>
 
 ### 期刊
 
-<Route author="Derekmini auto-bot-ty" example="/scitation/aapt/ajp" path="/scitation/:pub/:jrn" :paramsDesc="['出版社，URL 中 `scitation.org` 前部分','期刊，URL 中 `/toc/` 后部分']" radar="1" rssbud="1"/>
+<Route author="Derekmini auto-bot-ty" example="/scitation/aapt/ajp" path="/scitation/:pub/:jrn" :paramsDesc="['出版社，URL 中 `scitation.org` 前部分','期刊，URL 中 `/toc/` 后部分']" radar="1" rssbud="1" puppeteer="1"/>
 
 ### 专栏
 
-<Route author="Derekmini auto-bot-ty" example="/scitation/aapt/ajp/COMPUTATIONAL+PHYSICS" path="/scitation/:pub/:jrn/:sec" :paramsDesc="['出版社，URL 中 `scitation.org` 前部分','期刊，URL 中 `/toc/` 后部分','专栏，URL 中 `tocSection` 部分']" radar="1" rssbud="1"/>
+<Route author="Derekmini auto-bot-ty" example="/scitation/aapt/ajp/COMPUTATIONAL+PHYSICS" path="/scitation/:pub/:jrn/:sec" :paramsDesc="['出版社，URL 中 `scitation.org` 前部分','期刊，URL 中 `/toc/` 后部分','专栏，URL 中 `tocSection` 部分']" radar="1" rssbud="1" puppeteer="1"/>
 
 ## Springer
 

lib/v2/scitation/journal.js

@@ -1,10 +1,6 @@
-const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const path = require('path');
-const { art } = require('@/utils/render');
-
-const { CookieJar } = require('tough-cookie');
-const cookieJar = new CookieJar();
+const { puppeteerGet, renderDesc } = require('./utils');
+const config = require('@/config').value;
 
 module.exports = async (ctx) => {
     const pub = ctx.params.pub;
@@ -12,56 +8,47 @@ module.exports = async (ctx) => {
     const host = `https://${pub}.scitation.org`;
     const jrnlUrl = `${host}/toc/${jrn}/current?size=all`;
 
-    const response = await got(jrnlUrl, {
-        cookieJar,
-    });
-    const $ = cheerio.load(response.data);
-    const jrnlName = $('.header-journal-title').text();
-    const list = $('.card')
-        .map((_, item) => {
-            $(item).find('.access-text').remove();
-            const title = $(item).find('.hlFld-Title').text();
-            const authors = $(item).find('.entryAuthor.all').text();
-            const img = $(item).find('img').attr('src');
-            const link = $(item).find('.ref.nowrap').attr('href');
-            const doi = link.replace('/doi/full/', '');
-            return {
-                title,
-                link,
-                doi,
-                authors,
-                img,
-            };
-        })
-        .get();
+    // use Puppeteer due to the obstacle by cloudflare challenge
+    const browser = await require('@/utils/puppeteer')();
 
-    const renderDesc = (item) =>
-        art(path.join(__dirname, 'templates/description.art'), {
-            item,
-        });
-    const items = await Promise.all(
-        list.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                const response2 = await got.post(`${host}/action/PB2showAjaxAbstract`, {
-                    cookieJar,
-                    body: 'doi=' + item.doi,
-                    headers: {
-                        'X-Requested-With': 'XMLHttpRequest',
-                        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
-                        'Accept-Encoding': 'gzip, deflate, br',
-                    },
+    const { jrnlName, list } = await ctx.cache.tryGet(
+        jrnlUrl,
+        async () => {
+            const response = await puppeteerGet(jrnlUrl, browser);
+            const $ = cheerio.load(response);
+            const jrnlName = $('.header-journal-title').text();
+            const list = $('.card')
+                .toArray()
+                .map((item) => {
+                    $(item).find('.access-text').remove();
+                    const title = $(item).find('.hlFld-Title').text();
+                    const authors = $(item).find('.entryAuthor.all').text();
+                    const img = $(item).find('img').attr('src');
+                    const link = $(item).find('.ref.nowrap').attr('href');
+                    const doi = link.replace('/doi/full/', '');
+                    const description = renderDesc(title, authors, doi, img);
+                    return {
+                        title,
+                        link,
+                        doi,
+                        description,
+                    };
                 });
-                const $2 = cheerio.load(response2.data);
-                item.abstract = $2('span').text();
-                item.description = renderDesc(item);
-                return item;
-            })
-        )
+            return {
+                jrnlName,
+                list,
+            };
+        },
+        config.cache.routeExpire,
+        false
     );
 
+    browser.close();
+
     ctx.state.data = {
         title: jrnlName,
         link: jrnlUrl,
-        item: items,
+        item: list,
+        allowEmpty: true,
     };
 };

lib/v2/scitation/section.js

@@ -1,10 +1,6 @@
-const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const path = require('path');
-const { art } = require('@/utils/render');
-
-const { CookieJar } = require('tough-cookie');
-const cookieJar = new CookieJar();
+const { puppeteerGet, renderDesc } = require('./utils');
+const config = require('@/config').value;
 
 module.exports = async (ctx) => {
     const pub = ctx.params.pub;
@@ -13,17 +9,18 @@ module.exports = async (ctx) => {
     const host = `https://${pub}.scitation.org`;
     const jrnlUrl = `${host}/toc/${jrn}/current?size=all`;
 
-    const response = await got(jrnlUrl, {
-        cookieJar,
-    });
-    const $ = cheerio.load(response.data);
+    // use Puppeteer due to the obstacle by cloudflare challenge
+    const browser = await require('@/utils/puppeteer')();
+
+    const response = await puppeteerGet(jrnlUrl, browser);
+    const $ = cheerio.load(response);
     const jrnlName = $('.header-journal-title').text();
     const issueUrl = $('.row.js_issue.highlighted').find('a').attr('href');
     const section = $('.subject-heading')
-        .map((_, item) => ({
+        .toArray()
+        .map((item) => ({
             name: $(item).text(),
-        }))
-        .get();
+        }));
 
     let ifContainSection = false;
     for (let i = 0, l = section.length; i < l; i++) {
@@ -31,56 +28,41 @@ module.exports = async (ctx) => {
             ifContainSection = true;
         }
     }
+
     let list;
     if (ifContainSection === true) {
         const secUrl = `${issueUrl}?tocSection=${sec}`;
-        const response2 = await got(secUrl, {
-            cookieJar,
-        });
-        const $2 = cheerio.load(response2.data);
-        list = $2('.card')
-            .map((_, item) => {
-                $2(item).find('.access-text').remove();
-                const title = $2(item).find('.hlFld-Title').text();
-                const authors = $2(item).find('.entryAuthor.all').text();
-                const img = $2(item).find('img').attr('src');
-                const link = $2(item).find('.ref.nowrap').attr('href');
-                const doi = link.replace('/doi/full/', '');
-                return {
-                    title,
-                    link,
-                    doi,
-                    authors,
-                    img,
-                };
-            })
-            .get();
-
-        const renderDesc = (item) =>
-            art(path.join(__dirname, 'templates/description.art'), {
-                item,
-            });
-        list = await Promise.all(
-            list.map((item) =>
-                ctx.cache.tryGet(item.link, async () => {
-                    const response3 = await got.post(host.concat('/action/PB2showAjaxAbstract'), {
-                        cookieJar,
-                        body: 'doi=' + item.doi,
-                        headers: {
-                            'X-Requested-With': 'XMLHttpRequest',
-                            'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
-                            'Accept-Encoding': 'gzip, deflate, br',
-                        },
+        list = await ctx.cache.tryGet(
+            secUrl,
+            async () => {
+                const response2 = await puppeteerGet(secUrl, browser);
+                const $2 = cheerio.load(response2);
+                list = $2('.card')
+                    .toArray()
+                    .map((item) => {
+                        $2(item).find('.access-text').remove();
+                        const title = $2(item).find('.hlFld-Title').text();
+                        const authors = $2(item).find('.entryAuthor.all').text();
+                        const img = $2(item).find('img').attr('src');
+                        const link = $2(item).find('.ref.nowrap').attr('href');
+                        const doi = link.replace('/doi/full/', '');
+                        const description = renderDesc(title, authors, doi, img);
+                        return {
+                            title,
+                            link,
+                            doi,
+                            description,
+                        };
                     });
-                    const $3 = cheerio.load(response3.data);
-                    item.abstract = $3('span').text();
-                    item.description = renderDesc(item);
-                    return item;
-                })
-            )
+                return list;
+            },
+            config.cache.routeExpire,
+            false
         );
     }
 
+    browser.close();
+
     ctx.state.data = {
         title: jrnlName.concat(' - ', sec),
         link: jrnlUrl,

lib/v2/scitation/templates/description.art

@@ -1,11 +1,8 @@
 <p>
-    <span><big>{{ item.title }}</big></span><br>
+    <span><big>{{ title }}</big></span><br>
 </p>
 <p>
-    <span><small><i>{{ item.authors }}</i></small></span><br>
-    <span><small><i>https://doi.org/{{ item.doi }}</i></small></span><br>
-    {{ if item.img }}<img src={{ item.img }}>{{ /if }}
-</p>
-<p>
-    <span>{{ item.abstract }}</span><br>
+    <span><small><i>{{ authors }}</i></small></span><br>
+    <span><small><i>https://doi.org/{{ doi }}</i></small></span><br>
+    {{ if img }}<img src={{ img }}>{{ /if }}
 </p>

lib/v2/scitation/utils.js

@@ -0,0 +1,29 @@
+const path = require('path');
+const { art } = require('@/utils/render');
+
+const puppeteerGet = async (url, browser) => {
+    const page = await browser.newPage();
+    // await page.setExtraHTTPHeaders({ referer: host });
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' ? request.continue() : request.abort();
+    });
+    await page.goto(url, {
+        waitUntil: 'domcontentloaded',
+    });
+    const html = await page.evaluate(() => document.documentElement.innerHTML);
+    return html;
+};
+
+const renderDesc = (title, authors, doi, img) =>
+    art(path.join(__dirname, 'templates/description.art'), {
+        title,
+        authors,
+        doi,
+        img,
+    });
+
+module.exports = {
+    puppeteerGet,
+    renderDesc,
+};