feat(route): sina us stock and rollnews (#12493)

* refactor: migrate to v2

* refactor: generic rollnews

* docs: merge docs

* refactor: discovery use common utilities

* refactor: china finance use utilities

* refactor: csj use utils

* feat: us stock

* feat: add radar

* docs: mention default value

* docs: fix build
This commit is contained in:
Tony 2023-05-11 15:37:19 +01:00 committed by GitHub
parent 0267e2c307
commit d398081981
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 420 additions and 266 deletions

View File

@ -566,12 +566,6 @@ TokenInsight 官方亦有提供 RSS可参考 <https://api.tokeninsight.com/re
</Route>
## 新浪财经
### 新浪财经-国內
<Route author="yubinbai" example="/sina/finance" path="/sina/finance" />
## 雪球
### 今日话题

View File

@ -4454,11 +4454,61 @@ wechat-feeds 来源[已停止更新](https://github.com/hellodword/wechat-feeds/
<Route author="nczitzk" example="/news/whxw" path="/news/whxw"/>
## 新浪专栏
## 新浪
### 创事记
### 科技 - 科学探索
<Route author="xapool" example="/sina/csj" path="/sina/csj"/>
<Route author="LogicJake" example="/sina/discovery/zx" path="/sina/discovery/:type" :paramsDesc="['订阅分区类型,见下表']" radar="1">
| 最新 | 天文航空 | 动物植物 | 自然地理 | 历史考古 | 生命医学 | 生活百科 | 科技前沿 |
| ---- | -------- | -------- | -------- | -------- | -------- | -------- | -------- |
| zx | twhk | dwzw | zrdl | lskg | smyx | shbk | kjqy |
</Route>
### 滚动新闻
<Route author="xyqfer" example="/sina/rollnews" path="/sina/rollnews/:lid?" :paramsDesc="['分区 id可在 URL 中找到,默认为 `2509`']" radar="1">
| 全部 | 国内 | 国际 | 社会 | 体育 | 娱乐 | 军事 | 科技 | 财经 | 股市 | 美股 |
| ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
| 2509 | 2510 | 2511 | 2669 | 2512 | 2513 | 2514 | 2515 | 2516 | 2517 | 2518 |
</Route>
### 体育 - 综合
<Route author="nczitzk" example="/sina/sports/volley" path="/sina/sports/:type" :paramsDesc="['运动类型,见下表']" radar="1">
| 排球 | 游泳 | 乒乓球 | 羽毛球 | 台球 | 田径 | 体操 | 冰雪 | 射击 | 马术 | 拳击搏击 | UFC | 其他 |
| ------ | ---- | -------- | ------ | ------- | -------- | ----- | ------ | ---- | ----- | -------- | --- | ------ |
| volley | swim | pingpang | badmin | snooker | tianjing | ticao | winter | sh | horse | kungfu | ufc | others |
</Route>
### 财经-国內
<Route author="yubinbai" example="/sina/finance/china" path="/sina/finance/china/:lid?" :paramsDesc="['分区 id见下表默认为 `1686`']" radar="1">
| 国内滚动 | 宏观经济 | 金融新闻 | 地方经济 | 部委动态 | 今日财经 TOP10 |
| -------- | -------- | -------- | -------- | -------- | -------------- |
| 1686 | 1687 | 1690 | 1688 | 1689 | 3231 |
</Route>
### 美股
<Route author="TonyRL" example="/sina/finance/stock/usstock" path="/sina/finance/stock/usstock/:cids?" :paramsDesc="['分区 id见下表默认为 `57045`']" radar="1">
| 最新报道 | 中概股 | 国际财经 | 互联网 |
| -------- | ------ | -------- | ------ |
| 57045 | 57046 | 56409 | 40811 |
</Route>
### 专栏 - 创事记
<Route author="xapool" example="/sina/csj" path="/sina/csj" radar="1"/>
## 选股宝

View File

@ -2413,36 +2413,6 @@ category 对应的关键词有
<Route author="yhkang" example="/cztv/zjxwlb/daily" path="/cztv/zjxwlb/daily" />
## 新浪科技
### 科学探索
<Route author="LogicJake" example="/sina/discovery/zx" path="/sina/discovery/:type" :paramsDesc="['订阅分区类型']">
分类:
| zx | twhk | dwzw | zrdl | lskg | smyx | shbk | kjqy |
| ---- | -------- | -------- | -------- | -------- | -------- | -------- | -------- |
| 最新 | 天文航空 | 动物植物 | 自然地理 | 历史考古 | 生命医学 | 生活百科 | 科技前沿 |
</Route>
### 滚动新闻
<Route author="xyqfer" example="/sina/rollnews" path="/sina/rollnews" />
## 新浪体育
### 综合
<Route author="nczitzk" example="/sina/sports/volley" path="/sina/sports/:type" :paramsDesc="['运动类型,见下表']">
| 排球 | 游泳 | 乒乓球 | 羽毛球 | 台球 | 田径 | 体操 | 冰雪 | 射击 | 马术 | 拳击搏击 | UFC | 其他 |
| ------ | ---- | -------- | ------ | ------- | -------- | ----- | ------ | ---- | ----- | -------- | --- | ------ |
| volley | swim | pingpang | badmin | snooker | tianjing | ticao | winter | sh | mashu | kungfu | ufc | others |
</Route>
## 新唐人电视台
### 频道

View File

@ -1247,19 +1247,19 @@ router.get('/fitchratings/site/:type', lazyloadRouteHandler('./routes/fitchratin
// router.get('/mpaypass/main/:type?', lazyloadRouteHandler('./routes/mpaypass/main'));
// 新浪科技探索
router.get('/sina/discovery/:type', lazyloadRouteHandler('./routes/sina/discovery'));
// router.get('/sina/discovery/:type', lazyloadRouteHandler('./routes/sina/discovery'));
// 新浪科技滚动新闻
router.get('/sina/rollnews', lazyloadRouteHandler('./routes/sina/rollnews'));
// router.get('/sina/rollnews', lazyloadRouteHandler('./routes/sina/rollnews'));
// 新浪体育
router.get('/sina/sports/:type', lazyloadRouteHandler('./routes/sina/sports'));
// router.get('/sina/sports/:type', lazyloadRouteHandler('./routes/sina/sports'));
// 新浪专栏创事记
router.get('/sina/csj', lazyloadRouteHandler('./routes/sina/chuangshiji'));
// router.get('/sina/csj', lazyloadRouteHandler('./routes/sina/chuangshiji'));
// 新浪财经-国內
router.get('/sina/finance', lazyloadRouteHandler('./routes/sina/finance'));
// router.get('/sina/finance', lazyloadRouteHandler('./routes/sina/finance'));
// Animen
router.get('/animen/news/:type', lazyloadRouteHandler('./routes/animen/news'));

View File

@ -1,38 +0,0 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const url = 'http://feed.sina.com.cn/api/roll/get?pageid=402&lid=2559&num=20&versionNumber=1.2.8&page=1&encode=utf-8';
module.exports = async (ctx) => {
const response = await got.get(url);
const list = response.data.result.data;
const out = await Promise.all(
list.map(async (data) => {
const title = data.title;
const date = data.intime * 1000;
const link = data.url;
const description = await ctx.cache.tryGet(`sina-chuangshiji: ${link}`, async () => {
const response = await got.get(link);
const $ = cheerio.load(response.data);
return $('.blkContainerSblkCon').html();
});
const single = {
title,
link,
description,
pubDate: new Date(date).toUTCString(),
};
return Promise.resolve(single);
})
);
ctx.state.data = {
title: '新浪专栏-创事记',
link: 'https://tech.sina.com.cn/chuangshiji',
item: out,
};
};

View File

@ -1,52 +0,0 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const link = 'https://tech.sina.com.cn/discovery/';
const api = 'https://feed.sina.com.cn/api/roll/get?pageid=207&num=10&versionNumber=1.2.4&page=1&encode=utf-8&lid=';
const map = new Map([
['zx', { title: '最新', id: '1795' }],
['twhk', { title: '天文航空', id: '1796' }],
['dwzw', { title: '动物植物', id: '1797' }],
['zrdl', { title: '自然地理', id: '1798' }],
['lskg', { title: '历史考古', id: '1799' }],
['smyx', { title: '生命医学', id: '1800' }],
['shbk', { title: '生活百科', id: '1801' }],
['kjqy', { title: '科技前沿', id: '1802' }],
]);
module.exports = async (ctx) => {
const type = ctx.params.type;
const id = map.get(type).id;
const title = map.get(type).title;
const api_url = api + id;
const response = await got.get(api_url);
const list = response.data.result.data;
const out = await Promise.all(
list.map(async (data) => {
const title = data.title;
const date = data.intime * 1000;
const itemUrl = data.url;
const response = await got.get(itemUrl);
const $ = cheerio.load(response.data);
const description = $('.article').html();
const single = {
title,
link: itemUrl,
description,
pubDate: new Date(date).toUTCString(),
};
return Promise.resolve(single);
})
);
ctx.state.data = {
title: `${title}-新浪科技科学探索`,
link,
item: out,
};
};

View File

@ -1,37 +0,0 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
module.exports = async (ctx) => {
const url = 'http://feed.sina.com.cn/api/roll/get?pageid=155&lid=1686&num=20&versionNumber=1.2.8&page=1&encode=utf-8';
const response = await got.get(url);
const list = response.data.result.data;
const out = await Promise.all(
list.map(async (data) => {
const title = data.title;
const date = data.intime * 1000;
const link = data.url;
const description = await ctx.cache.tryGet(`sina-finance: ${link}`, async () => {
const response = await got.get(link);
const $ = cheerio.load(response.data);
return $('#artibody').html();
});
const single = {
title,
link,
description,
pubDate: new Date(date).toUTCString(),
};
return Promise.resolve(single);
})
);
ctx.state.data = {
title: '新浪财经-国內',
link: 'http://finance.sina.com.cn/china/',
item: out,
};
};

View File

@ -1,41 +0,0 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { parseDate } = require('@/utils/parse-date');
module.exports = async (ctx) => {
const response = await got(`https://feed.mix.sina.com.cn/api/roll/get?pageid=372&lid=2431&k=&num=50&page=1&r=${Math.random()}&callback=&_=${new Date().getTime()}`, {
headers: {
Referer: 'https://tech.sina.com.cn/',
},
});
const list = response.data.result.data;
const out = await Promise.all(
list.map(async (data) => {
const title = data.title;
const date = data.intime * 1000;
const link = data.url;
const description = await ctx.cache.tryGet(`sina-rollnews: ${link}`, async () => {
const response = await got.get(link);
const $ = cheerio.load(response.data);
return $('.article').html();
});
const single = {
title,
link,
description,
pubDate: parseDate(date),
};
return single;
})
);
ctx.state.data = {
title: '新浪科技滚动新闻',
link: 'https://tech.sina.com.cn/roll/rollnews.shtml#pageid=372&lid=2431&k=&num=50&page=1',
item: out,
};
};

View File

@ -1,54 +0,0 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
module.exports = async (ctx) => {
ctx.params.type = ctx.params.type || 'volley';
let currentUrl = `https://sports.sina.com.cn/others/${ctx.params.type}.shtml`,
query = 'ul.list2 li a';
if (ctx.params.type === 'ufc') {
currentUrl = 'http://roll.sports.sina.com.cn/s_ufc_all/index.shtml';
query = '#d_list ul li span a';
}
const response = await got({
method: 'get',
url: currentUrl,
});
const $ = cheerio.load(response.data);
const list = $(query)
.slice(0, 20)
.map((_, item) => {
item = $(item);
return {
link: item.attr('href'),
};
})
.get();
const items = await Promise.all(
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
});
const content = cheerio.load(detailResponse.data);
item.description = content('#artibody').html();
item.title = content('meta[property="og:title"]').attr('content');
item.pubDate = new Date(content('meta[property="article:published_time"]').attr('content')).toUTCString();
return item;
})
)
);
ctx.state.data = {
title: `${$('title').text().split('_')[0]} - 新浪体育`,
link: currentUrl,
item: items,
};
};

View File

@ -0,0 +1,17 @@
const { getRollNewsList, parseRollNewsList, parseArticle } = require('./utils');
module.exports = async (ctx) => {
const pageid = '402';
const lid = '2559';
const { limit = '50' } = ctx.query;
const response = await getRollNewsList(pageid, lid, limit);
const list = parseRollNewsList(response.data.result.data);
const out = await Promise.all(list.map((item) => parseArticle(item, ctx.cache.tryGet)));
ctx.state.data = {
title: '新浪专栏-创事记',
link: 'https://tech.sina.com.cn/chuangshiji',
item: out,
};
};

32
lib/v2/sina/discovery.js Normal file
View File

@ -0,0 +1,32 @@
const { getRollNewsList, parseRollNewsList, parseArticle } = require('./utils');
const link = 'https://tech.sina.com.cn/discovery/';
const map = new Map([
['zx', { title: '最新', id: '1795' }],
['twhk', { title: '天文航空', id: '1796' }],
['dwzw', { title: '动物植物', id: '1797' }],
['zrdl', { title: '自然地理', id: '1798' }],
['lskg', { title: '历史考古', id: '1799' }],
['smyx', { title: '生命医学', id: '1800' }],
['shbk', { title: '生活百科', id: '1801' }],
['kjqy', { title: '科技前沿', id: '1802' }],
]);
module.exports = async (ctx) => {
const type = ctx.params.type;
const lid = map.get(type).id;
const title = map.get(type).title;
const pageid = '207';
const { limit = '50' } = ctx.query;
const response = await getRollNewsList(pageid, lid, limit);
const list = parseRollNewsList(response.data.result.data);
const out = await Promise.all(list.map((item) => parseArticle(item, ctx.cache.tryGet)));
ctx.state.data = {
title: `${title}-新浪科技科学探索`,
link,
item: out,
};
};

View File

@ -0,0 +1,25 @@
const { getRollNewsList, parseRollNewsList, parseArticle } = require('../utils');
module.exports = async (ctx) => {
const map = {
1686: '国内滚动',
1687: '宏观经济',
1690: '金融新闻',
1688: '地方经济',
1689: '部委动态',
3231: '今日财经 TOP10',
};
const pageid = '155';
const { lid = '1686' } = ctx.params;
const { limit = '50' } = ctx.query;
const response = await getRollNewsList(pageid, lid, limit);
const list = parseRollNewsList(response.data.result.data);
const out = await Promise.all(list.map((item) => parseArticle(item, ctx.cache.tryGet)));
ctx.state.data = {
title: `新浪财经-${map[lid]}`,
link: 'http://finance.sina.com.cn/china/',
item: out,
};
};

View File

@ -0,0 +1,37 @@
const got = require('@/utils/got');
const { parseDate } = require('@/utils/parse-date');
const { parseArticle } = require('../../utils');
module.exports = async (ctx) => {
const { cids = '57045' } = ctx.params;
const { limit = '50' } = ctx.query;
const { data: response } = await got('https://interface.sina.cn/pc_api/public_news_data.d.json', {
searchParams: {
cids,
pdps: '',
smartFlow: '',
editLevel: '0,1,2,3',
type: 'std_news,std_slide,std_video',
pageSize: limit,
cTime: '1483200000',
tm: Date.now(),
up: '0',
action: '0',
_: new Date().getTime(),
},
});
const list = response.data.map((item) => ({
title: item.title,
link: item.url,
pubDate: parseDate(item.ctime, 'X'),
author: item.media,
}));
const items = await Promise.all(list.map((item) => parseArticle(item, ctx.cache.tryGet)));
ctx.state.data = {
title: '美股|美股行情|美股新闻 - 新浪财经',
link: 'https://finance.sina.com.cn/stock/usstock/',
item: items,
};
};

View File

@ -0,0 +1,8 @@
module.exports = {
'/csj': ['xapool'],
'/discovery/:type': ['LogicJake'],
'/finance/china/:lid?': ['yubinbai'],
'/finance/stock/usstock/:cids?': ['TonyRL'],
'/rollnews/:lid?': ['xyqfer'],
'/sports/:type?': ['nczitzk'],
};

49
lib/v2/sina/radar.js Normal file
View File

@ -0,0 +1,49 @@
module.exports = {
'sina.com.cn': {
_name: '新浪',
finance: [
{
title: '财经 - 国內',
docs: 'https://docs.rsshub.app/new-media.html#xin-lang',
source: ['/china', '/'],
target: '/sina/finance/china',
},
{
title: '美股',
docs: 'https://docs.rsshub.app/new-media.html#xin-lang',
source: ['/stock/usstock', '/'],
target: '/sina/finance/stock/usstock',
},
],
news: [
{
title: '滚动新闻',
docs: 'https://docs.rsshub.app/new-media.html#xin-lang',
source: ['/roll'],
target: (_, url) => `/sina/rollnews/${new URL(url).hash.match(/lid=(\d+)/)[1]}`,
},
],
sports: [
{
title: '体育 - 综合',
docs: 'https://docs.rsshub.app/new-media.html#xin-lang',
source: ['/others/:type', '/:type'],
target: (params) => `/sina/sports/${params.type}`,
},
],
tech: [
{
title: '专栏 - 创事记',
docs: 'https://docs.rsshub.app/new-media.html#xin-lang',
source: ['/chuangshiji', '/'],
target: '/sina/csj',
},
{
title: '科技 - 科学探索',
docs: 'https://docs.rsshub.app/new-media.html#xin-lang',
source: ['/discovery', '/'],
target: '/sina/discovery/zx',
},
],
},
};

31
lib/v2/sina/rollnews.js Normal file
View File

@ -0,0 +1,31 @@
const { getRollNewsList, parseRollNewsList, parseArticle } = require('./utils');
module.exports = async (ctx) => {
const map = {
2509: '全部',
2510: '国内',
2511: '国际',
2669: '社会',
2512: '体育',
2513: '娱乐',
2514: '军事',
2515: '科技',
2516: '财经',
2517: '股市',
2518: '美股',
};
const pageid = '153';
const { lid = '2509' } = ctx.params;
const { limit = '50' } = ctx.query;
const response = await getRollNewsList(pageid, lid, limit);
const list = parseRollNewsList(response.data.result.data);
const out = await Promise.all(list.map((item) => parseArticle(item, ctx.cache.tryGet)));
ctx.state.data = {
title: `新浪${map[lid]}滚动新闻`,
link: `https://news.sina.com.cn/roll/#pageid=${pageid}&lid=${lid}&k=&num=${limit}&page=1`,
item: out,
};
};

8
lib/v2/sina/router.js Normal file
View File

@ -0,0 +1,8 @@
module.exports = (router) => {
router.get('/csj', require('./chuangshiji'));
router.get('/discovery/:type', require('./discovery'));
router.get('/finance/china/:lid?', require('./finance/china'));
router.get('/finance/stock/usstock/:cids?', require('./finance/stock/usstock'));
router.get('/rollnews/:lid?', require('./rollnews'));
router.get('/sports/:type?', require('./sports'));
};

43
lib/v2/sina/sports.js Normal file
View File

@ -0,0 +1,43 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { parseArticle } = require('./utils');
module.exports = async (ctx) => {
const { type } = ctx.params;
let currentUrl = `https://sports.sina.com.cn/others/${type}.shtml`;
let query = 'ul.list2 li a';
if (type === 'ufc') {
currentUrl = 'http://roll.sports.sina.com.cn/s_ufc_all/index.shtml';
query = '#d_list ul li span a';
} else if (type === 'winter' || type === 'horse') {
currentUrl = `https://sports.sina.com.cn/${type}/`;
query = '[class^=news-list] .list li a';
}
const response = await got({
method: 'get',
url: currentUrl,
});
const $ = cheerio.load(response.data);
const list = $(query)
.toArray()
.map((item) => {
item = $(item);
return {
title: item.text(),
link: item.attr('href').replace('http://', 'https://'),
};
});
const items = await Promise.all(list.map((item) => parseArticle(item, ctx.cache.tryGet)));
ctx.state.data = {
title: `${$('title').text().split('_')[0]} - 新浪体育`,
description: $('meta[name="description"]').attr('content'),
link: currentUrl,
item: items,
};
};

View File

@ -0,0 +1,3 @@
{{ each slideData.images img }}
<img src="{{ img.download_img }}" alt="{{ img.intro }}">
{{ /each }}

View File

@ -0,0 +1,5 @@
{{ if videoUrl }}
<video controls poster="{{ poster }}" preload="none">
<source src="{{ videoUrl }}">
</video>
{{ /if }}

104
lib/v2/sina/utils.js Normal file
View File

@ -0,0 +1,104 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { parseDate } = require('@/utils/parse-date');
const timezone = require('@/utils/timezone');
const { art } = require('@/utils/render');
const { join } = require('path');
const getRollNewsList = (pageid, lid, limit) =>
got('https://feed.mix.sina.com.cn/api/roll/get', {
headers: {
referer: 'https://news.sina.com.cn/',
},
searchParams: {
pageid,
lid,
k: '',
num: limit,
page: 1,
r: Math.random(),
_: new Date().getTime(),
},
});
const parseRollNewsList = (data) =>
data.map((item) => ({
title: item.title,
description: item.intro,
link: item.url.replace('http://', 'https://'),
author: item.media_name,
pubDate: parseDate(item.intime, 'X'),
updated: parseDate(item.mtime, 'X'),
}));
const parseArticle = (item, tryGet) =>
tryGet(item.link, async () => {
const detailResponse = await got(item.link);
const $ = cheerio.load(detailResponse.data);
$('#left_hzh_ad, .appendQr_wrap, .app-kaihu-qr, .tech-quotation').remove();
const metaPublishTime = $('meta[property="article:published_time"]');
const htmlPubDate = $('#pub_date, .date');
const htmlDate = htmlPubDate.length ? timezone(parseDate(htmlPubDate.text(), ['YYYY年MM月DD日 HH:mm', 'YYYY年MM月DD日HH:mm']), 8) : null;
const metaDate = metaPublishTime.length ? parseDate(metaPublishTime.attr('content')) : htmlDate; // 2023-05-08T08:39:31+08:00
item.pubDate = item.pubDate ? item.pubDate : metaDate;
item.author = $('meta[property="article:author"]').attr('content');
if (item.link.startsWith('https://slide.sports.sina.com.cn/') || item.link.startsWith('https://slide.tech.sina.com.cn/')) {
const slideData = JSON.parse(
$('script')
.text()
.match(/var slide_data = (\{.*?\})\s/)[1]
);
item.description = art(join(__dirname, 'templates/slide.art'), { slideData });
} else if (item.link.startsWith('https://video.sina.com.cn/')) {
const videoId = $('script')
.text()
.match(/video_id:'?(.*?)'?,/)[1];
const { data: videoResponse } = await got('https://api.ivideo.sina.com.cn/public/video/play', {
searchParams: {
video_id: videoId,
appver: 'V11220.210521.03',
appname: 'sinaplayer_pc',
applt: 'web',
tags: 'sinaplayer_pc',
jsonp: '',
plid: 2021012801,
prid: '',
uid: '',
tid: '',
pid: 1,
ran: Math.random(),
r: item.link,
ssid: `gusr_pc_${Date.now()}`,
preload: 0,
uu: '',
isAuto: 1,
},
});
const videoData = videoResponse.data;
const poster = videoData.image;
const videoUrl = videoData.videos.find((v) => v.type === 'mp4').dispatch_result.url;
item.description = art(join(__dirname, 'templates/video.art'), { poster, videoUrl });
item.pubDate = parseDate(videoData.create_time, 'X');
} else if (item.link.startsWith('https://news.sina.com.cn/') || item.link.startsWith('https://mil.news.sina.com.cn/')) {
item.description = $('#article').html();
item.category = $('meta[name="keywords"]').attr('content').split(',');
} else {
// https://ent.sina.com.cn
// https://finance.sina.com.cn
// https://sports.sina.com.cn
item.description = $('#artibody').html();
item.category = $('#keywords').data('wbkey')?.split(',');
}
return item;
});
module.exports = {
getRollNewsList,
parseRollNewsList,
parseArticle,
};