From 193ef153ea0bf2ece5a280ed4ed6e7cc89f5700d Mon Sep 17 00:00:00 2001 From: Ethan Shen Date: Fri, 30 Oct 2020 17:23:07 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20add=20=E8=A7=82=E5=AF=9F=E8=80=85?= =?UTF-8?q?=E7=BD=91=E9=A6=96=E9=A1=B5=20(#5980)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/new-media.md | 26 +- lib/router.js | 12 +- lib/routes/guancha/index.js | 133 +++++++++ .../{guanchazhe => guancha}/personalpage.js | 0 lib/routes/{guanchazhe => guancha}/topic.js | 0 lib/routes/guanchazhe/index.js | 271 ------------------ 6 files changed, 162 insertions(+), 280 deletions(-) create mode 100644 lib/routes/guancha/index.js rename lib/routes/{guanchazhe => guancha}/personalpage.js (100%) rename lib/routes/{guanchazhe => guancha}/topic.js (100%) delete mode 100644 lib/routes/guanchazhe/index.js diff --git a/docs/new-media.md b/docs/new-media.md index 4ffa8b097..2b9381f60 100644 --- a/docs/new-media.md +++ b/docs/new-media.md @@ -729,19 +729,35 @@ area 分区选项 -## 观察者网 - 中国关怀 全球视野 +## 观察者网 -### 观察者首页 +### 首页 - + + +| 全部 | 评论 & 研究 | 要闻 | 风闻 | 热点新闻 | 滚动新闻 | +| ---- | ----------- | ----- | ------- | -------- | -------- | +| all | review | story | fengwen | redian | gundong | + +home = 评论 & 研究 + 要闻 + 风闻 + +others = 热点新闻 + 滚动新闻 + +::: tip 提示 + +观察者网首页左中右的三个 column 分别对应 **评论 & 研究**、**要闻**、**风闻** 三个部分。 + +::: + + ### 观察者风闻话题 - + ### 个人主页文章 - + ## 广告门 diff --git a/lib/router.js b/lib/router.js index 92df6c1a2..aadb3bb7b 100644 --- a/lib/router.js +++ b/lib/router.js @@ -1681,10 +1681,14 @@ router.get('/manhuadb/comics/:id', require('./routes/manhuadb/comics')); router.get('/zfrontier/postlist/:type', require('./routes/zfrontier/postlist')); router.get('/zfrontier/board/:boardId', require('./routes/zfrontier/board_postlist')); -// 观察者风闻话题 -router.get('/guanchazhe/topic/:id', require('./routes/guanchazhe/topic')); -router.get('/guanchazhe/personalpage/:uid', require('./routes/guanchazhe/personalpage')); -router.get('/guanchazhe/index/:type', require('./routes/guanchazhe/index')); +// 观察者网 +router.get('/guancha/topic/:id', require('./routes/guancha/topic')); +router.get('/guancha/personalpage/:uid', require('./routes/guancha/personalpage')); +router.get('/guancha/:caty?', require('./routes/guancha/index')); + +router.get('/guanchazhe/topic/:id', require('./routes/guancha/topic')); +router.get('/guanchazhe/personalpage/:uid', require('./routes/guancha/personalpage')); +router.get('/guanchazhe/index/:caty?', require('./routes/guancha/index')); // Hpoi 手办维基 router.get('/hpoi/info/:type?', require('./routes/hpoi/info')); diff --git a/lib/routes/guancha/index.js b/lib/routes/guancha/index.js new file mode 100644 index 000000000..18c5fbf0f --- /dev/null +++ b/lib/routes/guancha/index.js @@ -0,0 +1,133 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const date = require('@/utils/date'); + +const config = { + review: { title: '评论 & 研究', query: '.module-news-main' }, + story: { title: '要闻', query: '.img-List' }, + fengwen: { title: '风闻', query: '.fengwen-list' }, + redian: { title: '热点' }, + gundong: { title: '滚动' }, + all: { title: '全部' }, + home: { title: '首页' }, + others: { title: '热点 & 滚动' }, +}; + +module.exports = async (ctx) => { + const total = 10; + const caty = ctx.params.caty || 'all'; + const rootUrl = 'https://www.guancha.cn'; + + let newsList = [], + redianList = [], + gundongList = []; + + // 'review', 'story' and 'fengwen' come from homepage. + + if (caty === 'review' || caty === 'story' || caty === 'fengwen' || caty === 'all' || caty === 'home') { + const response = await got({ + method: 'get', + url: rootUrl, + }); + + const $ = cheerio.load(response.data); + + const fetchPost = (slice) => + slice + .find('h4.module-title a') + + // Filter some blank links which lead to no contents but 'https://user.guancha.cn'. + + .filter((_, item) => $(item).attr('href') !== 'https://user.guancha.cn') + .map((_, item) => { + item = $(item); + const link = item.attr('href'); + return { + title: item.text(), + link: `${(link.indexOf('http') === 0 ? '' : rootUrl) + link}`, + }; + }) + .get(); + + if (caty === 'all' || caty === 'home') { + newsList = fetchPost($(config.review.query)) + .slice(0, total / 3) + .concat(fetchPost($(config.story.query)).slice(0, total / 3), fetchPost($(config.fengwen.query)).slice(0, total / 3)); + } else { + newsList = fetchPost($(config[caty].query)).slice(0, total); + } + } + + // 'redian' and 'gundong' come from api. + + if (caty === 'redian' || caty === 'all' || caty === 'others') { + const response = await got({ + method: 'get', + url: `${rootUrl}/api/redian.htm`, + }); + redianList = response.data.items + .map((item) => ({ + title: item.TITLE, + link: `${rootUrl}${item.HTTP_URL}`, + })) + .slice(0, caty === 'all' ? total / 3 : total); + } + if (caty === 'gundong' || caty === 'all' || caty === 'others') { + const response = await got({ + method: 'get', + url: `${rootUrl}/api/gundong.htm`, + }); + gundongList = response.data.items + .map((item) => ({ + title: item.TITLE, + link: `${rootUrl}${item.HTTP_URL}`, + })) + .slice(0, caty === 'all' ? total / 3 : total); + } + + const items = await Promise.all( + newsList.concat(redianList, gundongList).map( + async (item) => + await ctx.cache.tryGet(item.link, async () => { + let detailResponse = await got({ + method: 'get', + url: item.link, + }); + + // Some links to posts in 'fengwen' must be redirected in order to fetch content. + // eg. https://www.guancha.cn/politics/2020_10_23_569021.shtml + // => https://user.guancha.cn/main/content?id=399176 + + const jumpMatch = detailResponse.data.match(/user.guancha.cn\/main\/content\?id=(.*)";/); + if (jumpMatch !== null) { + item.link = `https://user.guancha.cn/main/content?id=${jumpMatch[1]}`; + detailResponse = await got({ + method: 'get', + url: item.link, + }); + } + + const content = cheerio.load(detailResponse.data); + + const dateMatch = detailResponse.data.match(/"pubDate": "(.*)"/); + if (dateMatch === null) { + // PubDates of posts in 'fengwen' are in an informal format. + + item.pubDate = date(content('.time1').text(), 8); + } else { + item.pubDate = new Date(dateMatch[1]).toUTCString(); + } + + item.description = content('.all-txt').html() || content('.article-txt-content').html(); + + return item; + }) + ) + ); + + ctx.state.data = { + title: `观察者网 - ${config[caty].title}`, + link: rootUrl, + item: items, + }; +}; diff --git a/lib/routes/guanchazhe/personalpage.js b/lib/routes/guancha/personalpage.js similarity index 100% rename from lib/routes/guanchazhe/personalpage.js rename to lib/routes/guancha/personalpage.js diff --git a/lib/routes/guanchazhe/topic.js b/lib/routes/guancha/topic.js similarity index 100% rename from lib/routes/guanchazhe/topic.js rename to lib/routes/guancha/topic.js diff --git a/lib/routes/guanchazhe/index.js b/lib/routes/guanchazhe/index.js deleted file mode 100644 index 18209cd29..000000000 --- a/lib/routes/guanchazhe/index.js +++ /dev/null @@ -1,271 +0,0 @@ -const got = require('@/utils/got'); -const cheerio = require('cheerio'); -module.exports = async (ctx) => { - const type = ctx.params.type; - const host = 'https://www.guancha.cn'; - const ptype = { - all: { name: '首页Feeds', url: 'https://www.guancha.cn/' }, - redian: { name: '热点新闻', url: 'https://www.guancha.cn/api/redian.htm' }, - member: { name: '观察者', url: 'https://www.guancha.cn/api/member.htm' }, - gundong: { name: '滚动新闻', url: 'https://www.guancha.cn/api/new_gundong.htm' }, - }; - // 定义list存放请求数据,htmlarr暂存html数据组,outList存放输出的数据组 - const list = [], - htmlarr = [], - temp = [], - HomeFeeds = [], - RedianFeeds = [], - MemberFeeds = [], - GundongFeeds = []; - let AllFeeds = []; - // 获取新闻url过来的时间/ID - function getData(jscontent, option) { - switch (option) { - case 'date': { - const jsoutput = String(jscontent).substr(-23, 10).replace(/_/g, '-'); - return new Date(jsoutput).toLocaleDateString(); - } - case 'urlid': { - const jsoutput = String(jscontent).substr(-12, 6); - return jsoutput; - } - default: { - break; - } - } - } - // 循环获取ptype每个请求的数据内容 - for (const key in ptype) { - const link = ptype[key].url; - // eslint-disable-next-line no-await-in-loop - const response = await got({ - method: 'get', - url: link, - headers: { - Referer: host, - }, - }); - // 由于数据结构不太一致，将所有的数据内容抽离提取成统一的对象 - if (key === 'all') { - list[0] = response.data; - } - const $ = cheerio.load(list[0]); - switch (key) { - case 'all': - // 获取头条 + 清理不带有h4标题的li - htmlarr[0] = $('.content-headline').first(); - htmlarr[1] = $('.Review-item li h4').parent(); - htmlarr[2] = $('.img-List li h4').parent(); - $('.author-intro img').parent().remove(); - $('.module-img-head img').parent().remove(); - htmlarr[3] = $('[class="module-news gray"] .module-news-main').first().children(); - htmlarr[4] = $('[class="module-news-main mt15"]').children(); - temp.length = 0; - temp.push( - htmlarr[0].map((index, item) => { - item = $(item); - const herfString = item.find('h3>a').attr('href'); - const feed = { - category: ptype[key].name + '头条', - id: getData(herfString, 'urlid'), - title: item.find('h3').text().trim(), - pageurl: host + herfString, - picurl: item.find('a>img').first().attr('src'), - description: item.find('h3').text().trim() + ``, - date: getData(herfString, 'date'), - }; - return feed; - }) - ); - temp.push( - htmlarr[1].map((index, item) => { - item = $(item); - const herfString = item.find('h4.module-title>a').attr('href'); - const feed = { - category: ptype[key].name + '左1列', - id: getData(herfString, 'urlid'), - title: item.find('h4.module-title').text().trim(), - pageurl: host + herfString, - picurl: item.find('a.module-img>img').attr('src'), - description: - item.find('p.module-artile').text().trim() + - `

【全文阅读】`, - date: getData(herfString, 'date'), - }; - return feed; - }) - ); - temp.push( - htmlarr[2].map((index, item) => { - item = $(item); - const herfString = item.find('h4.module-title>a').attr('href'); - const feed = { - category: ptype[key].name + '中间+右边2列', - id: getData(herfString, 'urlid'), - title: item.find('h4.module-title').text().trim(), - pageurl: host + herfString, - picurl: item.find('a>img').attr('src'), - description: - item.find('.resemble-art').html() + - `

【全文阅读】`, - date: getData(herfString, 'date'), - }; - return feed; - }) - ); - temp.push( - htmlarr[3].map((index, item) => { - item = $(item); - const herfString = item.find('a').first().attr('href'); - const feed = { - category: ptype[key].name + '左列: 访谈、论坛', - id: getData(herfString, 'urlid'), - title: item.find('a').first().text(), - pageurl: host + herfString, - picurl: `0`, - description: item.find('a').first().text() + item.find('p').first().text() + `

【全文阅读】`, - date: getData(herfString, 'date'), - }; - return feed; - }) - ); - temp.push( - htmlarr[4].map((index, item) => { - item = $(item); - const herfString = item.find('a').first().attr('href'); - const feed = { - category: ptype[key].name + '右列: 历史、深度', - id: getData(herfString, 'urlid'), - title: item.find('a').first().text(), - pageurl: host + herfString, - picurl: `0`, - description: item.find('a').first().text() + `

【全文阅读】`, - date: getData(herfString, 'date'), - }; - return feed; - }) - ); - // 集结所有feeds对象给到HomeFeeds; - for (let i = 0; i < temp.length; i++) { - for (let k = 0; k < temp[i].length; k++) { - HomeFeeds.push(temp[i][k]); - } - } - break; - case 'redian': - list[1] = response.data.items; - temp.length = 0; - temp.push( - list[1].map((item) => ({ - category: ptype[key].name + '_热点', - id: '0', - title: '[热点]' + item.TITLE, - pageurl: host + '/' + item.HTTP_URL, - date: getData(item.HTTP_URL, 'date'), - picurl: '0', - })) - ); - // 集结所有feeds对象给到RedianFeeds; - for (let i = 0; i < temp.length; i++) { - for (let k = 0; k < temp[i].length; k++) { - RedianFeeds.push(temp[i][k]); - } - } - break; - case 'member': - list[2] = response.data.items; - temp.length = 0; - temp.push( - list[2].map((item) => ({ - category: ptype[key].name + '_右列观察员', - id: item.id, - title: '[付费]' + item.title, - pageurl: 'https://member.guancha.cn/post/view?id=' + item.id, - picurl: item.h_pic, - description: item.title + ``, - date: item.created_at, - })) - ); - // 集结所有feeds对象给到MemberFeeds; - for (let i = 0; i < temp.length; i++) { - for (let k = 0; k < temp[i].length; k++) { - MemberFeeds.push(temp[i][k]); - } - } - break; - case 'gundong': - list[3] = response.data.fenghot; - list[4] = response.data.member; - list[5] = response.data.kuaixun; - temp.length = 0; - temp.push( - list[5].map((item) => ({ - category: ptype[key].name + '_滚动新闻', - id: '0', - title: '[滚动新闻]' + item.TITLE, - pageurl: host + '/' + item.HTTP_URL, - picurl: '0', - date: getData(item.HTTP_URL, 'date'), - })) - ); - temp.push( - list[3].map((item) => ({ - category: ptype[key].name + '_风闻7天最热', - id: item.id, - title: '[风闻7天最热]' + item.title, - pageurl: 'https://user.guancha.cn/main/content?id=' + item.id, - picurl: '0', - })) - ); - temp.push( - list[4].map((item) => ({ - category: ptype[key].name + '_观察员', - id: item.id, - title: '[观察员付费]' + item.title, - pageurl: 'https://member.guancha.cn/post/view?id=' + item.id, - picurl: '0', - })) - ); - - for (let i = 0; i < temp.length; i++) { - for (let k = 0; k < temp[i].length; k++) { - GundongFeeds.push(temp[i][k]); - } - } - break; - } - } - // 集合所有数据,控制分类输出 - switch (type) { - case 'all': - AllFeeds = HomeFeeds.concat(RedianFeeds, MemberFeeds, GundongFeeds); - break; - case 'home': - AllFeeds = HomeFeeds; - break; - case 'others': - AllFeeds = RedianFeeds.concat(MemberFeeds, GundongFeeds); - break; - default: - AllFeeds = HomeFeeds; - break; - } - ctx.state.data = { - title: `观察者-首页新闻`, - link: host, - description: `观察者网，致力于荟萃中外思想者精华，鼓励青年学人探索，建中西文化交流平台，为崛起中的精英提供决策参考。`, - allowEmpty: true, - item: AllFeeds.map((item) => ({ - title: item.title, - description: item.description || item.title + `

【全文阅读】`, - pubDate: item.date || new Date().toLocaleDateString(), - link: item.pageurl, - category: item.category, - author: item.category, - })), - }; -};