From 4b7cc6c21ae4c36c60f329bc403378c85ddea3fc Mon Sep 17 00:00:00 2001 From: Haoyun Xia Date: Sun, 28 Jul 2024 19:35:50 +0800 Subject: [PATCH] fix(route/springer/journal): Update selectors in the web scraping code (#16278) --- lib/routes/springer/journal.ts | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/lib/routes/springer/journal.ts b/lib/routes/springer/journal.ts index 6a6a24284..e70cf16db 100644 --- a/lib/routes/springer/journal.ts +++ b/lib/routes/springer/journal.ts @@ -30,35 +30,34 @@ export const route: Route = { }, ], name: 'Journal', - maintainers: ['Derekmini', 'TonyRL'], + maintainers: ['Derekmini', 'TonyRL', 'xiahaoyun'], handler, }; async function handler(ctx) { - const host = 'https://www.springer.com'; + const host = 'https://link.springer.com'; const journal = ctx.req.param('journal'); - const jrnlUrl = `${host}/journal/${journal}`; + const jrnlUrl = `${host}/journal/${journal}/volumes-and-issues`; const response = await got(jrnlUrl, { cookieJar, }); const $ = load(response.data); - const jrnlName = $('h1#journalTitle').text().trim(); - const issueUrl = $('p.c-card__title.u-mb-16.u-flex-grow').find('a').attr('href'); + const jrnlName = $('span.app-journal-masthead__title').text().trim(); + const issueUrl = `${host}${$('li.c-list-group__item:first-of-type').find('a').attr('href')}`; const response2 = await got(issueUrl, { cookieJar, }); const $2 = load(response2.data); - const issue = $2('.app-volumes-and-issues__info').find('h1').text(); - const list = $2('article.c-card') + const issue = $2('h2.app-journal-latest-issue__heading').text(); + const list = $2('ol.u-list-reset > li') .map((_, item) => { - const title = $(item).find('.c-card__title').text().trim(); - const link = $(item).find('a').attr('href'); + const title = $(item).find('h3.app-card-open__heading').find('a').text().trim(); + const link = $(item).find('h3.app-card-open__heading').find('a').attr('href'); const doi = link.replace('https://link.springer.com/article/', ''); const img = $(item).find('img').attr('src'); const authors = $(item) - .find('.c-author-list') .find('li') .map((_, item) => $(item).text().trim()) .get() @@ -85,8 +84,7 @@ async function handler(ctx) { cookieJar, }); const $3 = load(response3.data); - $3('.c-article__sub-heading').remove(); - item.abstract = $3('div#Abs1-content').text(); + item.abstract = $3('div#Abs1-content > p:first-of-type').text(); item.description = renderDesc(item); return item; })