From 4340fe296bc21bbcecf563656bb65dddb042775a Mon Sep 17 00:00:00 2001 From: Enoch Ma Date: Thu, 21 Nov 2019 11:21:35 +0800 Subject: [PATCH] feat: add 10 routes (#3428) --- docs/en/multimedia.md | 2 +- docs/en/new-media.md | 33 ++++++++- docs/en/picture.md | 6 ++ docs/en/study.md | 40 +++++++++++ docs/new-media.md | 20 +++++- docs/picture.md | 6 ++ docs/study.md | 38 +++++++++- lib/router.js | 20 ++++++ lib/routes/1x/magazine.js | 48 +++++++++++++ lib/routes/60s-science/transcript.js | 3 +- lib/routes/elife/latest.js | 54 ++++++++++++++ lib/routes/elife/subject.js | 56 +++++++++++++++ lib/routes/nature/neuroscience/research.js | 65 +++++++++++++++++ lib/routes/nature/research.js | 64 +++++++++++++++++ lib/routes/pnas/latest.js | 58 +++++++++++++++ lib/routes/pnas/topic.js | 58 +++++++++++++++ lib/routes/quantamagazine/archive.js | 83 ++++++++++++++++++++++ lib/routes/simonsfoundation/articles.js | 53 ++++++++++++++ lib/routes/simonsfoundation/recommend.js | 45 ++++++++++++ 19 files changed, 744 insertions(+), 8 deletions(-) create mode 100644 lib/routes/1x/magazine.js create mode 100644 lib/routes/elife/latest.js create mode 100644 lib/routes/elife/subject.js create mode 100644 lib/routes/nature/neuroscience/research.js create mode 100644 lib/routes/nature/research.js create mode 100644 lib/routes/pnas/latest.js create mode 100644 lib/routes/pnas/topic.js create mode 100644 lib/routes/quantamagazine/archive.js create mode 100644 lib/routes/simonsfoundation/articles.js create mode 100644 lib/routes/simonsfoundation/recommend.js diff --git a/docs/en/multimedia.md b/docs/en/multimedia.md index 61fe12ef5..dac266a8f 100644 --- a/docs/en/multimedia.md +++ b/docs/en/multimedia.md @@ -8,7 +8,7 @@ pageClass: routes ### Transcript - + ## 99% Invisible diff --git a/docs/en/new-media.md b/docs/en/new-media.md index cc0a2caa8..06dd0e2a8 100644 --- a/docs/en/new-media.md +++ b/docs/en/new-media.md @@ -19,7 +19,7 @@ Supported sub-sites： ## AEON - + Subscribe it by channel： | Ideas | Essays | Videos | @@ -31,7 +31,9 @@ Subscribe it by subject or topic ： | ------- | ---------- | ---------- | ------- | ------- | | culture | philosophy | psychology | society | science | -Compared to the official one, the RSS feed generates by RSSHub not only has more fine-grained options, but eliminates pull quotes which can't be easily distinguished from other paragraphs by any RSS reader but purely disrupts the reading flow. Besides that, this feed also provides users a bio of the author in the end of the article. +Compared to the official one, the RSS feed generated by RSSHub not only has more fine-grained options, but eliminates pull quotes which can't be easily distinguished from other paragraphs by any RSS reader but purely disrupts the reading flow. Besides that, this feed also provides users a bio of the author in the end of the article. + + ## iDownloadBlog @@ -47,10 +49,35 @@ Provides a better reading experience (full text articles) over the official one. ### Topics - + This route provides a flexible plan with full text content to subscribe specific topic(s) on the Nautilus. Please visit [nautil.us](http://nautil.us) and click `Topics` to acquire whole topic list. + + +## Quanta Magazine + +### Archive + + + +Compared to the official one, this feed: + +1. supports LaTeX formulas, and +2. displays all pictures in the article (except those print-hidden multimedia materials). + + + +## Simons Foundation + +### Articles + + + +### What We’re Reading + + + ## The Verge diff --git a/docs/en/picture.md b/docs/en/picture.md index 1fde6e01d..fec8419b1 100644 --- a/docs/en/picture.md +++ b/docs/en/picture.md @@ -3,3 +3,9 @@ pageClass: routes --- # Picture + +## 1X + +### Magazine + + diff --git a/docs/en/study.md b/docs/en/study.md index 3f053e896..6a6c07fcd 100644 --- a/docs/en/study.md +++ b/docs/en/study.md @@ -4,6 +4,18 @@ pageClass: routes # Study +## eLife + +### Latest Research + +#### ALL + + + +#### Research by Subject + + + ## Google Scholar ### Keywords Monitoring @@ -21,3 +33,31 @@ Google Scholar has strict anti-crawling mechanism implemented, the demo below do 2. Advanced mode, visit [Google Scholar](https://scholar.google.com/schhp?hl=en&as_sdt=0,5), click the top left corner and select「Advanced Search」, fill in your conditions and submit the search. The URL should look like this: [https://scholar.google.com/scholar?as_q=data+visualization&as_epq=&as_oq=&as_eq=&as_occt=any&as_sauthors=&as_publication=&as_ylo=2018&as_yhi=&hl=en&as_sdt=0%2C5](https://scholar.google.com/scholar?as_q=data+visualization&as_epq=&as_oq=&as_eq=&as_occt=any&as_sauthors=&as_publication=&as_ylo=2018&as_yhi=&hl=en&as_sdt=0%2C5), copy everything after `https://scholar.google.com/scholar?` from the URL and use it as the query for this route. The complete URL for the above example should look like this: [https://rsshub.app/google/scholar/as_q=data+visualization&as_epq=&as_oq=&as_eq=&as_occt=any&as_sauthors=&as_publication=&as_ylo=2018&as_yhi=&hl=en&as_sdt=0%2C5](https://rsshub.app/google/scholar/as_q=data+visualization&as_epq=&as_oq=&as_eq=&as_occt=any&as_sauthors=&as_publication=&as_ylo=2018&as_yhi=&hl=en&as_sdt=0%2C5). + +## Nature + +### Latest Research + +#### Nature + + + +#### Nature Machine Intelligence + + + +#### Nature Neuroscience + + + +## Proceedings of The National Academy of Sciences + +### Latest Articles + +#### ALL + + + +#### Articles by Topic + + diff --git a/docs/new-media.md b/docs/new-media.md index bc4bfb5f4..c9a65c8d9 100644 --- a/docs/new-media.md +++ b/docs/new-media.md @@ -29,7 +29,7 @@ pageClass: routes ## AEON - + 支持以文体分类： | Ideas | Essays | Videos | @@ -41,6 +41,8 @@ pageClass: routes | ------- | ---------- | ---------- | ------- | ------- | | culture | philosophy | psychology | society | science | + + ## BOF ### 首页 @@ -137,6 +139,12 @@ pageClass: routes +## Quanta Magazine + +### 全部 + + + ## Readhub ### 分类 @@ -149,6 +157,16 @@ pageClass: routes +## Simons Foundation + +### 文章 + + + +### 推荐 + + + ## sixthtone ### 最新文章 diff --git a/docs/picture.md b/docs/picture.md index be65e5065..230c1cdb4 100644 --- a/docs/picture.md +++ b/docs/picture.md @@ -4,6 +4,12 @@ pageClass: routes # 图片 +## 1X + +### Magazine + + + ## Awesome Pigtails ### 最新图片 diff --git a/docs/study.md b/docs/study.md index 1ce8bf801..29cd8f89e 100644 --- a/docs/study.md +++ b/docs/study.md @@ -4,12 +4,46 @@ pageClass: routes # 学习 -## nature machine intelligence +## eLife -### latest research +### 最新成果 + +#### 综合 + + + +#### 细分领域 + + + +## Nature + +### Latest Research + +#### Nature + + + +#### Nature Machine Intelligence +#### Nature Neuroscience + + + +## Proceedings of The National Academy of Sciences + +### 最新文章 + +#### 全部 + + + +#### 根据领域分类 + + + ## X-MOL 平台 ### 新闻 diff --git a/lib/router.js b/lib/router.js index 7d0747e3f..b4de907be 100644 --- a/lib/router.js +++ b/lib/router.js @@ -30,6 +30,9 @@ router.get('/rsshub/rss', require('./routes/rsshub/rss')); // 1draw router.get('/1draw/', require('./routes/1draw/index')); +// 1X Magazine +router.get('/1x/magazine', require('./routes/1x/magazine')); + // bilibili router.get('/bilibili/user/video/:uid', require('./routes/bilibili/video')); router.get('/bilibili/user/article/:uid', require('./routes/bilibili/article')); @@ -1351,6 +1354,9 @@ router.get('/ps/list/:gridName', require('./routes/ps/list')); router.get('/ps/trophy/:id', require('./routes/ps/trophy')); router.get('/ps/ps4updates', require('./routes/ps/ps4updates')); +// Quanta Magazine +router.get('/quantamagazine/archive', require('./routes/quantamagazine/archive')); + // Nintendo router.get('/nintendo/eshop/jp', require('./routes/nintendo/eshop_jp')); router.get('/nintendo/eshop/hk', require('./routes/nintendo/eshop_hk')); @@ -1629,6 +1635,10 @@ router.get('/bof/home', require('./routes/bof/home')); router.get('/afdian/explore/:type?/:category?', require('./routes/afdian/explore')); router.get('/afdian/dynamic/:uid', require('./routes/afdian/dynamic')); +// Simons Foundation +router.get('/simonsfoundation/articles', require('./routes/simonsfoundation/articles')); +router.get('/simonsfoundation/recommend', require('./routes/simonsfoundation/recommend')); + // 王者荣耀 router.get('/pvp/newsindex/:type', require('./routes/pvp/newsindex')); @@ -1897,8 +1907,18 @@ router.get('/hatena/anonymous_diary/archive', require('./routes/hatena/anonymous router.get('/kaggle/discussion/:forumId/:sort?', require('./routes/kaggle/discussion')); router.get('/kaggle/competitions/:category?', require('./routes/kaggle/competitions')); +// eLife +router.get('/elife/latest', require('./routes/elife/latest')); +router.get('/elife/:tid', require('./routes/elife/subject')); + +// PNAS +router.get('/pnas/latest', require('./routes/pnas/latest')); +router.get('/pnas/:tid', require('./routes/pnas/topic')); + // nature router.get('/nature/natmachintell/research', require('./routes/nature/natmachintell/research')); +router.get('/nature/neuroscience/research', require('./routes/nature/neuroscience/research')); +router.get('/nature/research', require('./routes/nature/research')); // dlsite router.get('/dlsite/new/:type', require('./routes/dlsite/new')); diff --git a/lib/routes/1x/magazine.js b/lib/routes/1x/magazine.js new file mode 100644 index 000000000..3dc60c22a --- /dev/null +++ b/lib/routes/1x/magazine.js @@ -0,0 +1,48 @@ +const cheerio = require('cheerio'); +const got = require('@/utils/got'); + +function batchRemove($, array) { + array.forEach((element) => { + $(element).remove(); + }); +} + +module.exports = async (ctx) => { + const url = `https://1x.com/magazine`; + + const res = await got.get(url); + const $ = cheerio.load(res.data); + const unnecessary = ['#navigation', '.Headnote', '.Logotype', '.Footer', '.SplashImage-wrapper', '.Cta', '.CartSidebar', '.bottomNavigation']; + batchRemove($, unnecessary); + const list = $('a') + .slice(0, 10) + .get(); + const out = await Promise.all( + list.map(async (item) => { + const $ = cheerio.load(item); + const title = $(item).text(); + const partial = $(item).attr('href'); + const address = `https://1x.com${partial}`; + const cache = await ctx.cache.get(address); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + const res = await got.get(address); + const capture = cheerio.load(res.data); + const contents = capture('.PageSection.Static').html(); + const single = { + title, + description: contents, + link: address, + guid: address, + }; + ctx.cache.set(address, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + ctx.state.data = { + title: '1X Magazine', + link: url, + item: out, + }; +}; diff --git a/lib/routes/60s-science/transcript.js b/lib/routes/60s-science/transcript.js index 13e2d90a9..4e16ef68f 100644 --- a/lib/routes/60s-science/transcript.js +++ b/lib/routes/60s-science/transcript.js @@ -11,7 +11,8 @@ module.exports = async (ctx) => { const out = await Promise.all( list.map(async (item) => { const $ = cheerio.load(item); - const title = $(item).attr('aria-label'); + const fullTitle = $(item).attr('aria-label'); + const title = fullTitle.substring(0, fullTitle.length - 11); const address = $(item).attr('href'); const cache = await ctx.cache.get(address); if (cache) { diff --git a/lib/routes/elife/latest.js b/lib/routes/elife/latest.js new file mode 100644 index 000000000..ebbab7182 --- /dev/null +++ b/lib/routes/elife/latest.js @@ -0,0 +1,54 @@ +const cheerio = require('cheerio'); +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + const url = `https://elifesciences.org`; + + const res = await got.get(url); + const $ = cheerio.load(res.data); + $('.grid-secondary-column').remove(); + const list = $('li.listing-list__item').get(); + + const out = await Promise.all( + list.map(async (item) => { + const $ = cheerio.load(item); + const title = $('a.teaser__header_text_link').text(); + const partial = $('a.teaser__header_text_link').attr('href'); + const address = `${url}${partial}`; + const time = $('time').text(); + const description = $('div.teaser__body').text(); + const author = $('div.teaser__secondary_info').text(); + + const cache = await ctx.cache.get(address); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + const res = await got.get(address); + const capture = cheerio.load(res.data); + + const abstract = capture('#abstract .article-section__body').html(); + let contents; + if (description !== '') { + contents = `${description}

Abstract

${abstract}`; + } else { + contents = `
Abstract
${abstract}`; + } + + const single = { + title, + author: author, + description: contents, + link: address, + guid: address, + pubDate: new Date(time).toUTCString(), + }; + ctx.cache.set(address, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + ctx.state.data = { + title: `eLife | Latest Research`, + link: url, + item: out, + }; +}; diff --git a/lib/routes/elife/subject.js b/lib/routes/elife/subject.js new file mode 100644 index 000000000..c0aaf07c4 --- /dev/null +++ b/lib/routes/elife/subject.js @@ -0,0 +1,56 @@ +const cheerio = require('cheerio'); +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + const baseUrl = `https://elifesciences.org`; + const url = `${baseUrl}/subjects/${ctx.params.tid}`; + + const res = await got.get(url); + const $ = cheerio.load(res.data); + $('.grid-secondary-column').remove(); + const topic = $('h1.content-header__title.content-header__title--xx-short').text(); + const list = $('li.listing-list__item').get(); + + const out = await Promise.all( + list.map(async (item) => { + const $ = cheerio.load(item); + const title = $('a.teaser__header_text_link').text(); + const partial = $('a.teaser__header_text_link').attr('href'); + const address = `${baseUrl}${partial}`; + const time = $('time').text(); + const description = $('div.teaser__body').text(); + const author = $('div.teaser__secondary_info').text(); + + const cache = await ctx.cache.get(address); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + const res = await got.get(address); + const capture = cheerio.load(res.data); + + const abstract = capture('#abstract .article-section__body').html(); + let contents; + if (description !== '') { + contents = `${description}

Abstract

${abstract}`; + } else { + contents = `
Abstract
${abstract}`; + } + + const single = { + title, + author: author, + description: contents, + link: address, + guid: address, + pubDate: new Date(time).toUTCString(), + }; + ctx.cache.set(address, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + ctx.state.data = { + title: `eLife | ${topic}`, + link: url, + item: out, + }; +}; diff --git a/lib/routes/nature/neuroscience/research.js b/lib/routes/nature/neuroscience/research.js new file mode 100644 index 000000000..c125f6188 --- /dev/null +++ b/lib/routes/nature/neuroscience/research.js @@ -0,0 +1,65 @@ +const cheerio = require('cheerio'); +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + const url = `https://www.nature.com/neuro/research`; + + const res = await got.get(url); + const $ = cheerio.load(res.data); + const list = $('.border-bottom-1.pb20').get(); + + const out = await Promise.all( + list.map(async (item) => { + const $ = cheerio.load(item); + const title = $('h3 > a').text(); + const partial = $('h3 > a').attr('href'); + const address = `https://www.nature.com${partial}`; + const description = $('.hide-overflow.inline').text(); + const time = $('time').text(); + let author; + if ($('.js-list-authors-3 li').length > 3) { + author = + $('.js-list-authors-3 li') + .slice(0, 1) + .text() + ' et al.'; + } else { + author = $('.js-list-authors-3 li').text(); + } + const cache = await ctx.cache.get(address); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + const res = await got.get(address); + const capture = cheerio.load(res.data); + const abs = capture('div#Abs1-content.c-article-section__content').html(); + let contents; + if (description !== '' && abs !== null) { + contents = `${description}

Abstraction

${abs}

`; + } else { + if (abs !== null) { + contents = `
Abstraction
` + abs; + } + if (description !== '') { + contents = description; + } + } + + const single = { + title, + author: author, + description: contents, + link: address, + guid: address, + pubDate: new Date(time).toUTCString(), + }; + ctx.cache.set(address, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + ctx.state.data = { + title: `Nature Neuroscience | Latest Research`, + description: `Nature Neuroscience, a nature research journal`, + link: url, + item: out, + }; +}; diff --git a/lib/routes/nature/research.js b/lib/routes/nature/research.js new file mode 100644 index 000000000..f1fe91f77 --- /dev/null +++ b/lib/routes/nature/research.js @@ -0,0 +1,64 @@ +const cheerio = require('cheerio'); +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + const url = `https://www.nature.com/nature/research`; + + const res = await got.get(url); + const $ = cheerio.load(res.data); + const list = $('.border-bottom-1.pb20').get(); + + const out = await Promise.all( + list.map(async (item) => { + const $ = cheerio.load(item); + const title = $('h3 > a').text(); + const partial = $('h3 > a').attr('href'); + const address = `https://www.nature.com${partial}`; + const description = $('.hide-overflow.inline').text(); + const time = $('time').text(); + let author; + if ($('.js-list-authors-3 li').length > 3) { + author = + $('.js-list-authors-3 li') + .slice(0, 1) + .text() + ' et al.'; + } else { + author = $('.js-list-authors-3 li').text(); + } + const cache = await ctx.cache.get(address); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + const res = await got.get(address); + const capture = cheerio.load(res.data); + const abs = capture('div#Abs1-content.c-article-section__content').html(); + let contents; + if (description !== '' && abs !== null) { + contents = `${description}

Abstraction

${abs}

`; + } else { + if (abs !== null) { + contents = `
Abstraction
` + abs; + } + if (description !== '') { + contents = description; + } + } + + const single = { + title, + author: author, + description: contents, + link: address, + guid: address, + pubDate: new Date(time).toUTCString(), + }; + ctx.cache.set(address, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + ctx.state.data = { + title: `Nature | Latest Research`, + link: url, + item: out, + }; +}; diff --git a/lib/routes/pnas/latest.js b/lib/routes/pnas/latest.js new file mode 100644 index 000000000..5b96c751e --- /dev/null +++ b/lib/routes/pnas/latest.js @@ -0,0 +1,58 @@ +const cheerio = require('cheerio'); +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + const baseUrl = `https://www.pnas.org`; + const url = `${baseUrl}/content/early/recent`; + + const res = await got.get(url); + const $ = cheerio.load(res.data); + const list = $('.highwire-citation-pnas-list-complete').get(); + + const out = await Promise.all( + list.map(async (item) => { + const $ = cheerio.load(item); + const title = $('.highwire-cite-title').text(); + const partial = $('.highwire-cite-linked-title').attr('href'); + const address = `${baseUrl}${partial}`; + const time = $('time').text(); + let author; + if ($('.highwire-citation-authors span').length > 3) { + author = $('.highwire-citation-author.first').text() + ' et al.'; + } else { + author = $('.highwire-citation-authors span').text(); + } + const cache = await ctx.cache.get(address); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + const res = await got.get(address); + const capture = cheerio.load(res.data); + + const significance = capture('.executive-summary').html(); + const abstract = capture('.section.abstract').html(); + let contents; + if (abstract !== null) { + contents = significance + abstract; + } else { + contents = significance; + } + + const single = { + title, + author: author, + description: contents, + link: address, + guid: address, + pubDate: new Date(time).toUTCString(), + }; + ctx.cache.set(address, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + ctx.state.data = { + title: `PNAS | Latest Research`, + link: url, + item: out, + }; +}; diff --git a/lib/routes/pnas/topic.js b/lib/routes/pnas/topic.js new file mode 100644 index 000000000..3e26d65e8 --- /dev/null +++ b/lib/routes/pnas/topic.js @@ -0,0 +1,58 @@ +const cheerio = require('cheerio'); +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + const baseUrl = `https://www.pnas.org`; + const url = `${baseUrl}/content/by/section/${ctx.params.tid}`; + + const res = await got.get(url); + const $ = cheerio.load(res.data); + const list = $('.highwire-citation-pnas-list-complete').get(); + + const out = await Promise.all( + list.map(async (item) => { + const $ = cheerio.load(item); + const title = $('.highwire-cite-title').text(); + const partial = $('.highwire-cite-linked-title').attr('href'); + const address = `${baseUrl}${partial}`; + const time = $('time').text(); + let author; + if ($('.highwire-citation-authors span').length > 3) { + author = $('.highwire-citation-author.first').text() + ' et al.'; + } else { + author = $('.highwire-citation-authors span').text(); + } + const cache = await ctx.cache.get(address); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + const res = await got.get(address); + const capture = cheerio.load(res.data); + + const significance = capture('.executive-summary').html(); + const abstract = capture('.section.abstract').html(); + let contents; + if (abstract !== null) { + contents = significance + abstract; + } else { + contents = significance; + } + + const single = { + title, + author: author, + description: contents, + link: address, + guid: address, + pubDate: new Date(time).toUTCString(), + }; + ctx.cache.set(address, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + ctx.state.data = { + title: `PNAS | ${ctx.params.tid}`, + link: url, + item: out, + }; +}; diff --git a/lib/routes/quantamagazine/archive.js b/lib/routes/quantamagazine/archive.js new file mode 100644 index 000000000..7f292109b --- /dev/null +++ b/lib/routes/quantamagazine/archive.js @@ -0,0 +1,83 @@ +const cheerio = require('cheerio'); +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + const url = `https://feeder.co/discover/c1d270a6db/quantamagazine-org`; + + const res = await got.get(url); + const $ = cheerio.load(res.data); + const list = $('.card-title').get(); + + function batchRemove($, array) { + for (let index = 0; index < array.length; index++) { + $(array[index]).remove(); + } + } + + function batchRemoveAll($, array) { + for (let index = 0; index < array.length; index++) { + $(array[index]).each((index, item) => { + $(item).remove(); + }); + } + } + + const out = await Promise.all( + list.map(async (item) => { + const $ = cheerio.load(item); + const title = $('.discover-feed-item-link').text(); + const address = $('.discover-feed-item-link').attr('href'); + const cache = await ctx.cache.get(address); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + const res = await got.get(address); + const capture = cheerio.load(res.data); + + const infos = ['.post__footer', '.post__title__author-date']; + batchRemove(capture, infos); + + const author = capture('h3.mv05').text(); + const time = capture('.align-c.mb075 > p > em').text(); + + const unnecessary = ['.header-spacer', '.scale1.mha', '.post__title__author-date', '.post__aside--divider']; + batchRemove(capture, unnecessary); + + const disturbing = ['.hide-on-print', '.post__aside__pullquote', 'aside.post__sidebar.hide']; + batchRemoveAll(capture, disturbing); + + let contents = capture('#postBody').html(); + if (contents !== null) { + const latex = contents.replace(/\$latex([\s\S]+?)\$/g, ''); + contents = latex.replace(/
?/g, function(omit, src, cap) { + const imgUrl = src.replace(/\\([^un])/g, '$1'); + const img = ''; + + const noBS = cap.replace(/\\([^un])/g, '$1'); + const removeNL = noBS.replace(/\\n/g, ''); + const caption = removeNL.replace(/\\u([0-9]{1,3}[a-z][0-9]?|[0-9]{4}?)/g, function(omit, s) { + return String.fromCharCode(parseInt(s, 16)); + }); + const inset = '
' + img + '
' + caption + '
' + '
'; + return inset; + }); + } + + const single = { + title, + author: author, + description: contents, + link: address, + guid: address, + pubDate: new Date(time).toUTCString(), + }; + ctx.cache.set(address, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + ctx.state.data = { + title: 'Quanta Magazine', + link: `https://www.quantamagazine.org/`, + item: out, + }; +}; diff --git a/lib/routes/simonsfoundation/articles.js b/lib/routes/simonsfoundation/articles.js new file mode 100644 index 000000000..3b7444d81 --- /dev/null +++ b/lib/routes/simonsfoundation/articles.js @@ -0,0 +1,53 @@ +const cheerio = require('cheerio'); +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + const url = `https://www.simonsfoundation.org/news/articles`; + + const res = await got.get(url); + const $ = cheerio.load(res.data); + const list = $('.news-links-wrapper').get(); + + const out = await Promise.all( + list.map(async (item) => { + const $ = cheerio.load(item); + const title = $('a.news-title').text(); + const address = $('a.news-title').attr('href'); + const time = $('.news-date').text(); + + const cache = await ctx.cache.get(address); + if (cache) { + return Promise.resolve(JSON.parse(cache)); + } + + const res = await got.get(address); + const capture = cheerio.load(res.data); + const banner = capture('div.o-page-header__hero').html(); + const author = capture('.m-meta__inherit').text(); + capture('.m-block-info__animation').remove(); + capture('.o-blocks__left').remove(); + capture('.m-tags').remove(); + let contents; + if (banner !== null) { + contents = banner + capture('.o-blocks').html(); + } else { + contents = capture('.o-blocks').html(); + } + const single = { + title, + author: author, + description: contents, + link: address, + guid: address, + pubDate: new Date(time).toUTCString(), + }; + ctx.cache.set(address, JSON.stringify(single)); + return Promise.resolve(single); + }) + ); + ctx.state.data = { + title: 'Simons Foundation | Articles', + link: url, + item: out, + }; +}; diff --git a/lib/routes/simonsfoundation/recommend.js b/lib/routes/simonsfoundation/recommend.js new file mode 100644 index 000000000..378caeb93 --- /dev/null +++ b/lib/routes/simonsfoundation/recommend.js @@ -0,0 +1,45 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); + +const baseUrl = 'https://www.simonsfoundation.org/news/what-were-reading'; + +module.exports = async (ctx) => { + const response = await got({ + method: 'get', + url: baseUrl, + }); + + const $ = cheerio.load(response.data); + ctx.state.data = { + title: `Simons Foundation | What We're Reading`, + link: baseUrl, + description: `Simons Foundation - What We're Reading`, + item: $('.o-dated-list__row') + .map((index, item) => ({ + title: $(item) + .find('.m-post__title') + .text(), + description: $(item) + .find('.m-post__aside') + .html(), + pubDate: new Date( + $(item) + .find('.o-dated-list__day') + .text() + + ' ' + + $(item) + .find('.o-dated-list__month') + .text() + + ' ' + + $(item) + .find('.o-dated-list__year') + .text() + ).toUTCString(), + + link: $(item) + .find('.m-post__block-link') + .attr('href'), + })) + .get(), + }; +};