diff --git a/docs/en/journal.md b/docs/en/journal.md index dfd21809c..4667b4884 100644 --- a/docs/en/journal.md +++ b/docs/en/journal.md @@ -334,11 +334,11 @@ _only support Science Journal_ ### Journal - + ### Section - + ## Springer diff --git a/docs/journal.md b/docs/journal.md index 30261e9b3..eadcb5408 100644 --- a/docs/journal.md +++ b/docs/journal.md @@ -301,11 +301,11 @@ You can get all short name of a journal from ### 期刊 - + ### 专栏 - + ## Springer diff --git a/lib/v2/scitation/journal.js b/lib/v2/scitation/journal.js index 68ebe3efc..24430c937 100644 --- a/lib/v2/scitation/journal.js +++ b/lib/v2/scitation/journal.js @@ -1,10 +1,6 @@ -const got = require('@/utils/got'); const cheerio = require('cheerio'); -const path = require('path'); -const { art } = require('@/utils/render'); - -const { CookieJar } = require('tough-cookie'); -const cookieJar = new CookieJar(); +const { puppeteerGet, renderDesc } = require('./utils'); +const config = require('@/config').value; module.exports = async (ctx) => { const pub = ctx.params.pub; @@ -12,56 +8,47 @@ module.exports = async (ctx) => { const host = `https://${pub}.scitation.org`; const jrnlUrl = `${host}/toc/${jrn}/current?size=all`; - const response = await got(jrnlUrl, { - cookieJar, - }); - const $ = cheerio.load(response.data); - const jrnlName = $('.header-journal-title').text(); - const list = $('.card') - .map((_, item) => { - $(item).find('.access-text').remove(); - const title = $(item).find('.hlFld-Title').text(); - const authors = $(item).find('.entryAuthor.all').text(); - const img = $(item).find('img').attr('src'); - const link = $(item).find('.ref.nowrap').attr('href'); - const doi = link.replace('/doi/full/', ''); - return { - title, - link, - doi, - authors, - img, - }; - }) - .get(); + // use Puppeteer due to the obstacle by cloudflare challenge + const browser = await require('@/utils/puppeteer')(); - const renderDesc = (item) => - art(path.join(__dirname, 'templates/description.art'), { - item, - }); - const items = await Promise.all( - list.map((item) => - ctx.cache.tryGet(item.link, async () => { - const response2 = await got.post(`${host}/action/PB2showAjaxAbstract`, { - cookieJar, - body: 'doi=' + item.doi, - headers: { - 'X-Requested-With': 'XMLHttpRequest', - 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', - 'Accept-Encoding': 'gzip, deflate, br', - }, + const { jrnlName, list } = await ctx.cache.tryGet( + jrnlUrl, + async () => { + const response = await puppeteerGet(jrnlUrl, browser); + const $ = cheerio.load(response); + const jrnlName = $('.header-journal-title').text(); + const list = $('.card') + .toArray() + .map((item) => { + $(item).find('.access-text').remove(); + const title = $(item).find('.hlFld-Title').text(); + const authors = $(item).find('.entryAuthor.all').text(); + const img = $(item).find('img').attr('src'); + const link = $(item).find('.ref.nowrap').attr('href'); + const doi = link.replace('/doi/full/', ''); + const description = renderDesc(title, authors, doi, img); + return { + title, + link, + doi, + description, + }; }); - const $2 = cheerio.load(response2.data); - item.abstract = $2('span').text(); - item.description = renderDesc(item); - return item; - }) - ) + return { + jrnlName, + list, + }; + }, + config.cache.routeExpire, + false ); + browser.close(); + ctx.state.data = { title: jrnlName, link: jrnlUrl, - item: items, + item: list, + allowEmpty: true, }; }; diff --git a/lib/v2/scitation/section.js b/lib/v2/scitation/section.js index 507a91bf7..75779f484 100644 --- a/lib/v2/scitation/section.js +++ b/lib/v2/scitation/section.js @@ -1,10 +1,6 @@ -const got = require('@/utils/got'); const cheerio = require('cheerio'); -const path = require('path'); -const { art } = require('@/utils/render'); - -const { CookieJar } = require('tough-cookie'); -const cookieJar = new CookieJar(); +const { puppeteerGet, renderDesc } = require('./utils'); +const config = require('@/config').value; module.exports = async (ctx) => { const pub = ctx.params.pub; @@ -13,17 +9,18 @@ module.exports = async (ctx) => { const host = `https://${pub}.scitation.org`; const jrnlUrl = `${host}/toc/${jrn}/current?size=all`; - const response = await got(jrnlUrl, { - cookieJar, - }); - const $ = cheerio.load(response.data); + // use Puppeteer due to the obstacle by cloudflare challenge + const browser = await require('@/utils/puppeteer')(); + + const response = await puppeteerGet(jrnlUrl, browser); + const $ = cheerio.load(response); const jrnlName = $('.header-journal-title').text(); const issueUrl = $('.row.js_issue.highlighted').find('a').attr('href'); const section = $('.subject-heading') - .map((_, item) => ({ + .toArray() + .map((item) => ({ name: $(item).text(), - })) - .get(); + })); let ifContainSection = false; for (let i = 0, l = section.length; i < l; i++) { @@ -31,56 +28,41 @@ module.exports = async (ctx) => { ifContainSection = true; } } + let list; if (ifContainSection === true) { const secUrl = `${issueUrl}?tocSection=${sec}`; - const response2 = await got(secUrl, { - cookieJar, - }); - const $2 = cheerio.load(response2.data); - list = $2('.card') - .map((_, item) => { - $2(item).find('.access-text').remove(); - const title = $2(item).find('.hlFld-Title').text(); - const authors = $2(item).find('.entryAuthor.all').text(); - const img = $2(item).find('img').attr('src'); - const link = $2(item).find('.ref.nowrap').attr('href'); - const doi = link.replace('/doi/full/', ''); - return { - title, - link, - doi, - authors, - img, - }; - }) - .get(); - - const renderDesc = (item) => - art(path.join(__dirname, 'templates/description.art'), { - item, - }); - list = await Promise.all( - list.map((item) => - ctx.cache.tryGet(item.link, async () => { - const response3 = await got.post(host.concat('/action/PB2showAjaxAbstract'), { - cookieJar, - body: 'doi=' + item.doi, - headers: { - 'X-Requested-With': 'XMLHttpRequest', - 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', - 'Accept-Encoding': 'gzip, deflate, br', - }, + list = await ctx.cache.tryGet( + secUrl, + async () => { + const response2 = await puppeteerGet(secUrl, browser); + const $2 = cheerio.load(response2); + list = $2('.card') + .toArray() + .map((item) => { + $2(item).find('.access-text').remove(); + const title = $2(item).find('.hlFld-Title').text(); + const authors = $2(item).find('.entryAuthor.all').text(); + const img = $2(item).find('img').attr('src'); + const link = $2(item).find('.ref.nowrap').attr('href'); + const doi = link.replace('/doi/full/', ''); + const description = renderDesc(title, authors, doi, img); + return { + title, + link, + doi, + description, + }; }); - const $3 = cheerio.load(response3.data); - item.abstract = $3('span').text(); - item.description = renderDesc(item); - return item; - }) - ) + return list; + }, + config.cache.routeExpire, + false ); } + browser.close(); + ctx.state.data = { title: jrnlName.concat(' - ', sec), link: jrnlUrl, diff --git a/lib/v2/scitation/templates/description.art b/lib/v2/scitation/templates/description.art index c55ee2a9b..3f368b9b1 100644 --- a/lib/v2/scitation/templates/description.art +++ b/lib/v2/scitation/templates/description.art @@ -1,11 +1,8 @@

- {{ item.title }}
+ {{ title }}

- {{ item.authors }}
- https://doi.org/{{ item.doi }}
- {{ if item.img }}{{ /if }} -

-

- {{ item.abstract }}
+ {{ authors }}
+ https://doi.org/{{ doi }}
+ {{ if img }}{{ /if }}

\ No newline at end of file diff --git a/lib/v2/scitation/utils.js b/lib/v2/scitation/utils.js new file mode 100644 index 000000000..988a82683 --- /dev/null +++ b/lib/v2/scitation/utils.js @@ -0,0 +1,29 @@ +const path = require('path'); +const { art } = require('@/utils/render'); + +const puppeteerGet = async (url, browser) => { + const page = await browser.newPage(); + // await page.setExtraHTTPHeaders({ referer: host }); + await page.setRequestInterception(true); + page.on('request', (request) => { + request.resourceType() === 'document' ? request.continue() : request.abort(); + }); + await page.goto(url, { + waitUntil: 'domcontentloaded', + }); + const html = await page.evaluate(() => document.documentElement.innerHTML); + return html; +}; + +const renderDesc = (title, authors, doi, img) => + art(path.join(__dirname, 'templates/description.art'), { + title, + authors, + doi, + img, + }); + +module.exports = { + puppeteerGet, + renderDesc, +};