xiaoxue_admin
/
RSSHub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(route): caixin blog (#11695) 

* fix(route): caixin blog

* fix: url
This commit is contained in:
Tony 2023-01-25 12:41:36 -08:00 committed by GitHub
parent a09a38d3dd
commit d10fb2334c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 405 additions and 286 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docs/traditional-media.md
View File

@ -586,7 +586,7 @@ Provides all of the articles by the specified Yahoo! author.
### 新闻分类
<Route author="idealclover" example="/caixin/finance/regulation" path="/caixin/:column/:category" :paramsDesc="['栏目名', '栏目下的子分类名']">
<Route author="idealclover" example="/caixin/finance/regulation" path="/caixin/:column/:category" :paramsDesc="['栏目名', '栏目下的子分类名']" supportPodcast="1">
Column 列表：
@ -610,11 +610,11 @@ Category 列表：
### 首页新闻
<Route author="EsuRt" example="/caixin/article" path="/caixin/article"/>
<Route author="EsuRt" example="/caixin/article" path="/caixin/article" radar="1" supportPodcast="1"/>
### 最新文章
<Route author="tpnonthealps" example="/caixin/latest" path="/caixin/latest">
<Route author="tpnonthealps" example="/caixin/latest" path="/caixin/latest" radar="1">
说明：此 RSS feed 会自动抓取财新网的最新文章，但不包含 FM 及视频内容。
@ -622,11 +622,11 @@ Category 列表：
### 财新数据通
<Route author="nczitzk" example="/caixin/database" path="/caixin/database"/>
<Route author="nczitzk" example="/caixin/database" path="/caixin/database" radar="1"/>
### 财新一线
<Route author="boypt" example="/caixin/yxnews" path="/caixin/yxnews"/>
<Route author="boypt" example="/caixin/k" path="/caixin/k" radar="1" supportPodcast="1"/>
## 参考消息

12
lib/router.js
View File

@ -288,11 +288,11 @@ router.get('/mihoyo/bh2/:type', lazyloadRouteHandler('./routes/mihoyo/bh2'));
// router.get('/cctv-special/:id?', lazyloadRouteHandler('./routes/cctv/special'));
// 财新博客
router.get('/caixin/blog/:column', lazyloadRouteHandler('./routes/caixin/blog'));
router.get('/caixin/article', lazyloadRouteHandler('./routes/caixin/article'));
router.get('/caixin/database', lazyloadRouteHandler('./routes/caixin/database'));
router.get('/caixin/yxnews', lazyloadRouteHandler('./routes/caixin/yxnews'));
router.get('/caixin/:column/:category', lazyloadRouteHandler('./routes/caixin/category'));
// router.get('/caixin/blog/:column', lazyloadRouteHandler('./routes/caixin/blog'));
// router.get('/caixin/article', lazyloadRouteHandler('./routes/caixin/article'));
// router.get('/caixin/database', lazyloadRouteHandler('./routes/caixin/database'));
// router.get('/caixin/yxnews', lazyloadRouteHandler('./routes/caixin/yxnews'));
// router.get('/caixin/:column/:category', lazyloadRouteHandler('./routes/caixin/category'));
// 草榴社区
router.get('/t66y/post/:tid', lazyloadRouteHandler('./routes/t66y/post'));
@ -3633,7 +3633,7 @@ router.get('/cppcc/:slug?', lazyloadRouteHandler('./routes/gov/cppcc/index'));
router.get('/nace/blog/:sort?', lazyloadRouteHandler('./routes/nace/blog'));
// Caixin Latest
router.get('/caixin/latest', lazyloadRouteHandler('./routes/caixin/latest'));
// router.get('/caixin/latest', lazyloadRouteHandler('./routes/caixin/latest'));
// Semiconductor Industry Association
router.get('/semiconductors/latest-news', lazyloadRouteHandler('./routes/semiconductors/latest-news'));

46
lib/routes/caixin/article.js
View File

@ -1,46 +0,0 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
module.exports = async (ctx) => {
const response = await got({
method: 'get',
url: 'http://mapiv5.caixin.com//m/api/getWapIndexListByPage?page=1&callback=&_=1560140003179',
headers: {
Referer: `http://mapiv5.caixin.com/`,
Host: 'mapiv5.caixin.com',
},
});
const data = response.data.data.list;
const items = await Promise.all(
data.map(async (item) => {
const link = item.web_url;
const summary = `<blockquote><p>${item.summary}</p></blockquote><img src="${item.pics}">`;
const key = 'caixin_article_' + link;
const fullText = await ctx.cache.tryGet(key, async () => {
const result = await got.get(link);
const $ = cheerio.load(result.data);
return $('div#Main_Content_Val.text').html();
});
return {
title: item.title,
description: fullText ? summary + fullText : summary,
link,
pubDate: new Date(item.time * 1000),
author: item.author_name,
};
})
);
ctx.state.data = {
title: `财新网 - 首页`,
link: `http://www.caixin.com/`,
description: '财新网 - 首页',
item: items,
};
};

62
lib/routes/caixin/blog.js
View File

@ -1,62 +0,0 @@
const got = require('@/utils/got');
const parser = require('@/utils/rss-parser');
const cheerio = require('cheerio');
const { isValidHost } = require('@/utils/valid-host');
async function load(link, need_feed_description) {
const response = await got.get(link);
const $ = cheerio.load(response.data);
const article = $('.blog_content').removeAttr('style');
article.find('img').removeAttr('style');
article
.find('div')
// Non-breaking space U+00A0, `&nbsp;` in html
// element.children[0].data === $(element, article).text()
.filter((_, element) => element.children[0].data === String.fromCharCode(160))
.remove();
const description = article.html();
const item = { description };
if (need_feed_description) {
const author = $('div.widget.author_detail p:nth-child(2)');
// workaround to split author name and author article
const author_name = author.find('strong');
author_name.text(author_name.text() + '，');
item.feed_description = author.text();
}
return item;
}
module.exports = async (ctx) => {
const { column } = ctx.params;
if (!isValidHost(column)) {
throw Error('Invalid column');
}
const link = `http://${column}.blog.caixin.com`;
const feed_url = `${link}/feed`;
const feed = await parser.parseURL(feed_url);
const title = `财新博客 - ${/[^»]*$/.exec(feed.title)[0]}`;
const items = await Promise.all(
feed.items.slice(0, 10).map(async (item, index) => {
const link = item.link;
const single = {
title: item.title,
pubDate: item.pubDate,
link,
author: item['dc:creator'],
};
const other = await ctx.cache.tryGet(link, () => load(link, index === 0));
return Promise.resolve(Object.assign({}, single, other));
})
);
ctx.state.data = {
title,
link,
description: items[0].feed_description,
item: items,
};
};

48
lib/routes/caixin/category.js
View File

@ -1,48 +0,0 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { isValidHost } = require('@/utils/valid-host');
module.exports = async (ctx) => {
const category = ctx.params.category;
const column = ctx.params.column;
const url = `http://${column}.caixin.com/${category}`;
if (!isValidHost(column)) {
throw Error('Invalid column');
}
const response = await got({
method: 'get',
url,
headers: {
Referer: url,
},
});
const $ = cheerio.load(response.data);
const title = $('title').text();
const list = $('.stitXtuwen_list dl dd');
const items = [];
for (let i = 0; i < list.length; i++) {
const item = {
title: $(list[i]).children('h4').children().text(),
desc: $(list[i]).children('p').text(),
pubDate: new Date($(list[i]).children('span').text().replace('年', '-').replace('月', '-').replace('日', '')).toUTCString(),
url: $(list[i]).children('h4').children().attr('href'),
};
items.push(item);
}
ctx.state.data = {
title,
link: url,
description: '财新网 - 提供财经新闻及资讯服务',
item: items.map((item) => ({
title: item.title,
description: item.desc,
guid: item.title,
link: item.url,
pubDate: item.pubDate,
})),
};
};

75
lib/routes/caixin/latest.js
View File

@ -1,75 +0,0 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { parseDate } = require('@/utils/parse-date');
const timezone = require('@/utils/timezone');
module.exports = async (ctx) => {
const li_r = await got({
method: 'get',
url: 'https://www.caixin.com/2021-08-22/101758426.html',
});
const $ = cheerio.load(li_r.data);
const list = $('div.columnBox a[name="new_artical"]~ul.list li');
const tasks = [];
list.map((_index, li) => $(li).find('a').first().attr('href'))
.filter((_index, link) => !link.includes('fm.caixin.com') && !link.includes('video.caixin.com') && !link.includes('datanews.caixin.com')) // content filter
.each((_index, link) => {
const entry = ctx.cache.tryGet(link, async () => {
const entry_r = await got.get(link);
const $ = cheerio.load(entry_r.data);
// title
const h1 = $('div#conTit h1').text();
// desc items
const subhead = $('div#subhead.subhead').text();
let pic_url = $('img.cx-img-loader').attr('src');
if (pic_url === undefined) {
pic_url = $('img.cx-img-loader').attr('data-src');
}
const pic_alt = $('dl.media_pic dd').text();
const content = $('div#Main_Content_Val.text').html();
/*
const [, count] =
$('a.box_titlecontent.cost-uibtn')
.text()
.match(/本文共计(\d+)字/) || [];
if (count !== 0 && count !== undefined) {
content = `${content}<blockquote><p>此乃财新通收费文章，全文共计${count}字。</p></blockquote>`;
}
*/
// desc
let desc;
if (pic_url === undefined) {
desc = `<blockquote><p>${subhead}</p></blockquote>${content}`;
} else {
desc = `<blockquote><p>${subhead}</p></blockquote><img src="${pic_url}" alt="${pic_alt}">${content}`;
}
// time
/*
const [, year, month, day, hour, minute] = $('div#artInfo.artInfo')
.text()
.match(/(\d+)年(\d+)月(\d+)日 *(\d+):(\d+)/);
*/
return {
title: h1,
description: desc,
pubDate: timezone(parseDate($('div#artInfo.artInfo').text(), 'YYYY年MM月DD日 HH:mm'), +8),
link,
};
});
tasks.push(entry);
});
const rss = await Promise.all(tasks);
ctx.state.data = {
title: '财新网 - 最新文章',
link: 'http://www.caixin.com/',
item: rss,
};
};

30
lib/routes/caixin/yxnews.js
View File

@ -1,30 +0,0 @@
const got = require('@/utils/got');
module.exports = async (ctx) => {
const response = await got({
method: 'get',
url: 'http://k.caixin.com/app/v1/list?productIdList=8,28&uid=&unit=1&name=&code=&deviceType=&device=&userTag=&p=1&c=20',
headers: {
Referer: `http://k.caixin.com/web/`,
Host: 'k.caixin.com',
},
});
const data = response.data.data.list;
const items = await Promise.all(
data.map((item) => ({
title: item.title,
description: item.text,
link: `http://k.caixin.com/web/detail_${item.oneline_news_code}`,
pubDate: new Date(item.ts),
author: '财新一线',
}))
);
ctx.state.data = {
title: `财新网 - 财新一线新闻`,
link: `http://k.caixin.com/`,
description: `财新网 - 财新一线新闻`,
item: items,
};
};

27
lib/v2/caixin/article.js Normal file
View File

@ -0,0 +1,27 @@
const got = require('@/utils/got');
const { parseDate } = require('@/utils/parse-date');
const { parseArticle } = require('./utils');
module.exports = async (ctx) => {
const { data: response } = await got('https://mapiv5.caixin.com/m/api/getWapIndexListByPage');
const list = response.data.list.map((item) => ({
title: item.title,
description: item.summary,
author: item.author_name,
pubDate: parseDate(item.time, 'X'),
link: item.web_url,
pics: item.pics,
audio: item.cms_audio_url,
audio_image_url: item.audio_image_url,
}));
const items = await Promise.all(list.map((item) => parseArticle(item, ctx.cache.tryGet)));
ctx.state.data = {
title: '财新网 - 首页',
link: 'https://www.caixin.com',
description: '财新网 - 首页',
item: items,
};
};

58
lib/v2/caixin/blog.js Normal file
View File

@ -0,0 +1,58 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { isValidHost } = require('@/utils/valid-host');
const { parseDate } = require('@/utils/parse-date');
const { parseBlogArticle } = require('./utils');
module.exports = async (ctx) => {
const { column } = ctx.params;
if (!isValidHost(column)) {
throw Error('Invalid column');
}
const link = `https://${column}.blog.caixin.com`;
const { data: response } = await got(link);
const $ = cheerio.load(response);
const authorId = $('script[type="text/javascript"]')
.text()
.match(/window\.authorId = (\d+);/)[1];
const authorName = $('script[type="text/javascript"]')
.text()
.match(/window\.authorName = "(.*)";/)[1];
const avatar = $('script[type="text/javascript"]')
.text()
.match(/avatar: "(.*?)",/)[1];
const introduce = $('script[type="text/javascript"]')
.text()
.match(/introduce: "(.*?)"\n/)[1];
const {
data: { data },
} = await got('https://blog.caixin.com/blog-api/post/posts', {
searchParams: {
page: 1,
size: ctx.query.limit ? parseInt(ctx.query.limit) : 20,
content: '',
authorId,
sort: 'publishTime',
direction: 'DESC',
},
});
const posts = data.map((item) => ({
title: item.title,
description: item.brief,
author: item.displayName,
link: item.guid.replace('http://', 'https://'),
pubDate: parseDate(item.publishTime, 'x'),
}));
const items = await Promise.all(posts.map((item) => parseBlogArticle(item, ctx.cache.tryGet)));
ctx.state.data = {
title: `财新博客 - ${authorName}`,
link,
description: introduce,
image: avatar,
item: items,
};
};

56
lib/v2/caixin/category.js Normal file
View File

@ -0,0 +1,56 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { isValidHost } = require('@/utils/valid-host');
const { parseDate } = require('@/utils/parse-date');
const timezone = require('@/utils/timezone');
const { parseArticle } = require('./utils');
module.exports = async (ctx) => {
const category = ctx.params.category;
const column = ctx.params.column;
const url = `https://${column}.caixin.com/${category}`;
if (!isValidHost(column)) {
throw Error('Invalid column');
}
const response = await got(url);
const $ = cheerio.load(response.data);
const title = $('head title').text();
const entity = JSON.parse(
$('script')
.text()
.match(/var entity = (\{.*?\})/)[1]
);
const {
data: { datas: data },
} = await got('https://gateway.caixin.com/api/extapi/homeInterface.jsp', {
searchParams: {
subject: entity.id,
type: 0,
count: ctx.query.limit ? parseInt(ctx.query.limit) : 25,
picdim: '_266_177',
start: 0,
},
});
const list = data.map((item) => ({
title: item.desc,
description: item.summ,
link: item.link.replace('http://', 'https://'),
pubDate: timezone(parseDate(item.time), +8),
category: item.keyword.split(' '),
audio: item.audioUrl,
audio_image_url: item.pict.imgs[0].url,
}));
const items = await Promise.all(list.map((item) => parseArticle(item, ctx.cache.tryGet)));
ctx.state.data = {
title,
link: url,
description: '财新网 - 提供财经新闻及资讯服务',
item: items,
};
};

28
lib/routes/caixin/database.js → lib/v2/caixin/database.js
View File

@ -2,38 +2,38 @@ const got = require('@/utils/got');
const cheerio = require('cheerio');
const timezone = require('@/utils/timezone');
const { parseDate } = require('@/utils/parse-date');
const { art } = require('@/utils/render');
const path = require('path');
module.exports = async (ctx) => {
const rootUrl = 'https://database.caixin.com';
const currentUrl = `${rootUrl}/news`;
const response = await got({
method: 'get',
url: currentUrl,
});
const currentUrl = `${rootUrl}/news/`;
const response = await got(currentUrl);
const $ = cheerio.load(response.data);
const list = $('h4 a')
.map((_, item) => {
.toArray()
.map((item) => {
item = $(item);
return {
title: item.text(),
link: item.attr('href'),
link: item.attr('href').replace('http://', 'https://'),
};
})
.get();
});
const items = await Promise.all(
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const detailResponse = await got(item.link);
const content = cheerio.load(detailResponse.data);
item.pubDate = timezone(parseDate(content('#pubtime_baidu').text()), +8);
item.description = `<p>${content('#subhead').html()}</p>`;
item.description = art(path.join(__dirname, 'templates/article.art'), {
item,
$: content,
});
item.author = content('.top-author').text();
return item;
})

40
lib/v2/caixin/k.js Normal file
View File

@ -0,0 +1,40 @@
const got = require('@/utils/got');
const { parseDate } = require('@/utils/parse-date');
module.exports = async (ctx) => {
const response = await got('https://k.caixin.com/app/v1/list', {
searchParams: {
productIdList: '8,28',
uid: '',
unit: 1,
name: '',
code: '',
deviceType: '',
device: '',
userTag: '',
p: 1,
c: 20,
},
});
const data = response.data.data.list;
const items = data.map((item) => {
const hasAudio = item.audio_url || Object.values(item.audios)[0];
return {
title: item.title,
description: item.text,
link: `http://k.caixin.com/web/detail_${item.oneline_news_code}`,
pubDate: parseDate(item.ts, 'x'),
author: '财新一线',
enclosure_url: hasAudio ? item.audio_url || Object.values(item.audios)[0] : undefined,
enclosure_type: hasAudio ? 'audio/mpeg' : undefined,
};
});
ctx.state.data = {
title: '财新网 - 财新一线新闻',
link: 'https://k.caixin.com/web/',
description: '财新网 - 财新一线新闻',
item: items,
};
};

50
lib/v2/caixin/latest.js Normal file
View File

@ -0,0 +1,50 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { parseDate } = require('@/utils/parse-date');
const timezone = require('@/utils/timezone');
const { art } = require('@/utils/render');
const path = require('path');
module.exports = async (ctx) => {
const li_r = await got('https://www.caixin.com/2021-08-22/101758426.html');
const $ = cheerio.load(li_r.data);
const list = $('div.columnBox a[name="new_artical"]~ul.list li')
.toArray()
.map((li) => ({
link: $(li).find('a').first().attr('href').replace('http://', 'https://'),
}))
.filter((item) => !item.link.startsWith('https://fm.caixin.com/') && !item.link.startsWith('https://video.caixin.com/') && !item.link.startsWith('https://datanews.caixin.com/')); // content filter
const rss = await Promise.all(
list.map((item) =>
ctx.cache.tryGet(`caixin:latest:${item.link}`, async () => {
const entry_r = await got(item.link);
const $ = cheerio.load(entry_r.data);
// title
const h1 = $('div#conTit h1').text();
// desc
const desc = art(path.join(__dirname, 'templates/article.art'), {
item,
$,
});
item.title = h1;
item.description = desc;
item.pubDate = timezone(parseDate($('#pubtime_baidu').text()), +8);
// prevent cache coliision with /caixin/article and /caixin/:column/:category
// since those have podcasts
item.guid = `caixin:latest:${item.link}`;
return item;
})
)
);
ctx.state.data = {
title: '财新网 - 最新文章',
link: 'https://www.caixin.com/',
item: rss,
};
};

8
lib/v2/caixin/maintainer.js Normal file
View File

@ -0,0 +1,8 @@
module.exports = {
'/article': ['EsuRt'],
'/blog/:column': ['Maecenas'],
'/database': ['nczitzk'],
'/k': ['boypt'],
'/latest': ['tpnonthealps'],
'/:column/:category': ['idealclover'],
};

43
lib/v2/caixin/radar.js Normal file
View File

@ -0,0 +1,43 @@
module.exports = {
'caixin.com': {
_name: '财新网',
'.': [
{
title: '新闻分类',
docs: 'https://docs.rsshub.app/traditional-media.html#cai-xin-wang',
},
{
title: '首页新闻',
docs: 'https://docs.rsshub.app/traditional-media.html#cai-xin-wang',
source: ['/'],
target: '/caixin/article',
},
{
title: '最新文章',
docs: 'https://docs.rsshub.app/traditional-media.html#cai-xin-wang',
source: ['/'],
target: '/caixin/latest',
},
{
title: '用户博客',
docs: 'https://docs.rsshub.app/blog.html#cai-xin-bo-ke',
},
],
database: [
{
title: '财新数据通',
docs: 'https://docs.rsshub.app/traditional-media.html#cai-xin-wang',
source: ['/news', '/'],
target: '/caixin/database',
},
],
k: [
{
title: '财新一线',
docs: 'https://docs.rsshub.app/traditional-media.html#cai-xin-wang',
source: ['/web', '/'],
target: '/caixin/database',
},
],
},
};

8
lib/v2/caixin/router.js Normal file
View File

@ -0,0 +1,8 @@
module.exports = (router) => {
router.get('/article', require('./article'));
router.get('/blog/:column', require('./blog'));
router.get('/database', require('./database'));
router.get('/k', require('./k'));
router.get('/latest', require('./latest'));
router.get('/:column/:category', require('./category'));
};

38
lib/v2/caixin/templates/article.art Normal file
View File

@ -0,0 +1,38 @@
{{ if item.audio }}
<audio src="{{ item.audio }}" controls></audio><br>
{{ /if }}
{{ if $('.article .subhead').length }}
<blockquote><p>{{@ $('.article .subhead').html() }}</p></blockquote>
<br>
{{ /if }}
{{ if $('.article .media').length }}
{{@ $('.article .media').html() }}
<br>
{{ /if }}
{{ if $('.article .content_video').length }}
<% const video = $('script').text().match(/initPlayer\('(.*?)','(.*?)'\)/); %>
<% const videoUrl = video[1]; %>
<% const poster = video[2]; %>
<video controls preload="none" poster="{{ poster }}" src="{{ videoUrl }}"></video>
<br>
{{ /if }}
{{ if $('div#Main_Content_Val.text').length }}
{{@ $('div#Main_Content_Val.text').html() }}
{{ else }}
{{ if item.summary }}
<blockquote><p>{{ item.summary }}</p></blockquote>
<br>
{{ /if }}
{{ if item.pics?.includes('#') }}
{{ each item.pics.split('#') pic }}
<img src="{{ pic }}">
<br>
{{ /each }}
{{ else }}
<img src="{{ item.pics }}">
<br>
{{ /if }}
{{ /if }}

52
lib/v2/caixin/utils.js Normal file
View File

@ -0,0 +1,52 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { art } = require('@/utils/render');
const path = require('path');
const parseArticle = (item, tryGet) => {
if (new URL(item.link).hostname.match(/\.blog\.caixin\.com$/)) {
return parseBlogArticle(item, tryGet);
} else {
return tryGet(item.link, async () => {
const { data: response } = await got(item.link);
const $ = cheerio.load(response);
item.description = art(path.join(__dirname, 'templates/article.art'), {
item,
$,
});
if (item.audio) {
item.itunes_item_image = item.audio_image_url;
item.enclosure_url = item.audio;
item.enclosure_type = 'audio/mpeg';
}
return item;
});
}
};
const parseBlogArticle = (item, tryGet) =>
tryGet(item.link, async () => {
const response = await got(item.link);
const $ = cheerio.load(response.data);
const article = $('#the_content').removeAttr('style');
article.find('img').removeAttr('style');
article
.find('p')
// Non-breaking space U+00A0, `&nbsp;` in html
// element.children[0].data === $(element, article).text()
.filter((_, element) => element.children[0].data === String.fromCharCode(160))
.remove();
item.description = article.html();
return item;
});
module.exports = {
parseArticle,
parseBlogArticle,
};