fix(route): Yahoo News (#16099)
* fix * clean up * make codefactor happy * fix description
This commit is contained in:
WilliamGates
GitHub
parent
136e994cef
commit
88040a7609
|
|
@ -0,0 +1,148 @@
|
|||
import { Route } from '@/types';
|
||||
import cache from '@/utils/cache';
|
||||
import parser from '@/utils/rss-parser';
|
||||
import { getArchive, getCategories, parseList, parseItem } from './utils';
|
||||
import InvalidParameterError from '@/errors/types/invalid-parameter';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/news/:region/:category?',
|
||||
categories: ['new-media'],
|
||||
example: '/yahoo/news/hk/world',
|
||||
parameters: {
|
||||
region: 'Region, `hk/tw/au/ca/fr/malaysia/nz/sg/uk/en(us)`, the part represented by the asterisk (*) in *.news.yahoo.com',
|
||||
category: 'Category, The part represented by the asterisk (*) in .news.yahoo.com/rss/*, region "hk/tw" differs, see the description below',
|
||||
},
|
||||
features: {
|
||||
requireConfig: false,
|
||||
requirePuppeteer: false,
|
||||
antiCrawler: false,
|
||||
supportBT: false,
|
||||
supportPodcast: false,
|
||||
supportScihub: false,
|
||||
},
|
||||
radar: [
|
||||
{
|
||||
source: ['news.yahoo.com/'],
|
||||
},
|
||||
],
|
||||
name: 'News',
|
||||
maintainers: ['KeiLongW', 'williamgateszhao'],
|
||||
handler,
|
||||
url: 'news.yahoo.com/',
|
||||
description: `
|
||||
\`Region\`
|
||||
|
||||
Support all regions represented by the asterisk (*) in *.news.yahoo.com, such as hk/tw/au/ca/fr/malaysia/nz/sg/uk/en(us). For www.yahoo.com, use en or us. Sites with news domains other than *.news.yahoo.com, such as de.nachrichten.yahoo.com or news.yahoo.co.jp, are not supported.
|
||||
|
||||
\`Category\`
|
||||
|
||||
The parsing method for Yahoo Hong Kong and Taiwan is quite unique. All supported categories are as follows
|
||||
|
||||
Category for hk.news.yahoo.com (hongkong)
|
||||
|
||||
| 全部 | 港聞 | 兩岸國際 | 財經 | 娛樂 | 體育 | 健康 | 親子 | 副刊 |
|
||||
| ------- | --------- | -------- | -------- | ------------- | ------ | ------ | --------- | ---------- |
|
||||
| (empty) | hong-kong | world | business | entertainment | sports | health | parenting | supplement |
|
||||
|
||||
Category for tw.news.yahoo.com (taiwan)
|
||||
|
||||
| 全部 | 政治 | 財經 | 娛樂 | 運動 | 社會地方 | 國際 | 生活 | 健康 | 科技 | 品味 |
|
||||
| ------- | -------- | ------- | ------------- | ------ | -------- | ----- | --------- | ------ | ---------- | ----- |
|
||||
| (empty) | politics | finance | entertainment | sports | society | world | lifestyle | health | technology | style |
|
||||
|
||||
Other Yahoo news is fetched from the RSS provided by Yahoo. Please refer to the categories displayed on the pages of *.news.yahoo.com (for example, "world"), and try to access *.news.yahoo.com/rss/world to see if it is accessible and contains recent news (some categories exist but are not updated). If it is accessible and has recent news, then that category can be used on the corresponding site. For example, the available categories for news.yahoo.com are as follows
|
||||
|
||||
Category for news.yahoo.com (US)
|
||||
|
||||
| All | US | Politics | World | Science | Tech |
|
||||
| ------- | -- | -------- | ----- | ------- | ---- |
|
||||
| (empty) | us | politics | world | science | tech |
|
||||
|
||||
To give another example, since uk.news.yahoo.com/rss/ukoriginal is accessible and has recent news, /yahoo/news/uk/ukoriginal is a valid RSSHub route.
|
||||
|
||||
\`author\`
|
||||
|
||||
For Yahoo Hong Kong and Yahoo Taiwan, please use another "news source" route.
|
||||
|
||||
For other Yahoo News, this route's RSS provides the author field. You can use RSSHub's built-in "content filtering" feature. For example, /yahoo-wg/news/tw/technology?filter_author=Yahoo%20Tech|Engadget can filter out news with authors containing Yahoo Tech or Engadget from Yahoo Taiwan's technology news, which is the Chinese version of Engadget.
|
||||
`,
|
||||
zh: {
|
||||
name: '新闻',
|
||||
description: `
|
||||
\`区域 Region\`
|
||||
|
||||
支持所有 *.news.yahoo.com 中*号所代表的区域, 例如\`hk/tw/au/ca/fr/malaysia/nz/sg/uk/en(us)\`, 其中 www.yahoo.com 用 en 或 us 来表示。不支持新闻域名不为 *.news.yahoo.com 的站点如 de.nachrichten.yahoo.com 或 news.yahoo.co.jp。
|
||||
|
||||
\`分类 Category\`
|
||||
|
||||
香港和台湾雅虎的读取方式比较特别, 所有支持的 category 如下
|
||||
|
||||
hk.news.yahoo.com (香港) 所支持的分类
|
||||
|
||||
| 全部 | 港聞 | 兩岸國際 | 財經 | 娛樂 | 體育 | 健康 | 親子 | 副刊 |
|
||||
| ------- | --------- | -------- | -------- | ------------- | ------ | ------ | --------- | ---------- |
|
||||
| （留空） | hong-kong | world | business | entertainment | sports | health | parenting | supplement |
|
||||
|
||||
tw.news.yahoo.com (台湾) 所支持的分类
|
||||
|
||||
| 全部 | 政治 | 財經 | 娛樂 | 運動 | 社會地方 | 國際 | 生活 | 健康 | 科技 | 品味 |
|
||||
| ------- | -------- | ------- | ------------- | ------ | -------- | ----- | --------- | ------ | ---------- | ----- |
|
||||
| （留空） | politics | finance | entertainment | sports | society | world | lifestyle | health | technology | style |
|
||||
|
||||
其他雅虎新闻读取自 yahoo 提供的 RSS, 请根据 *.news.yahoo.com 的页面上展示的分类(例如 world ), 尝试 *.news.yahoo.com/rss/world 能否访问并且有近期的新闻(有些分类存在但未更新), 如果可以的话则该分类可以用在相应站点, 例如 news.yahoo.com 可用的分类如下
|
||||
|
||||
news.yahoo.com (美国) 所支持的分类
|
||||
|
||||
| All | US | Politics | World | Science | Tech |
|
||||
| ------- | -- | -------- | ----- | ------- | ---- |
|
||||
| (留空) | us | politics | world | science | tech |
|
||||
|
||||
再举例, 由于 uk.news.yahoo.com/rss/ukoriginal 可以访问并且有较新的新闻, 所以 /yahoo/news/uk/ukoriginal 是一个有效的RSSHub路由。
|
||||
|
||||
\`作者 author\`
|
||||
|
||||
对于香港和台湾雅虎, 请使用另一个"新聞來源"路由。
|
||||
|
||||
对于其他雅虎新闻, 本路由的 RSS 中提供了 author 字段, 可使用 RSSHub 的内置"内容过滤"功能, 例如 /yahoo-wg/news/tw/technology?filter_author=Yahoo%20Tech|Engadget 可从台湾雅虎的科技新闻中过滤出作者名称中包含 Yahoo Tech 或者 Engadget 的新闻, 即瘾科技中文版。
|
||||
`,
|
||||
},
|
||||
};
|
||||
|
||||
async function handler(ctx) {
|
||||
const region = ['en', 'EN', 'us', 'US', 'www', 'WWW', ''].includes(ctx.req.param('region')) ? '' : ctx.req.param('region').toLowerCase();
|
||||
const category = ctx.req.param('category');
|
||||
if (!['hk', 'tw', 'au', 'ca', 'fr', 'malaysia', 'nz', 'sg', 'uk', ''].includes(region)) {
|
||||
throw new InvalidParameterError(`Unknown region: ${region}`);
|
||||
}
|
||||
|
||||
const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 20;
|
||||
|
||||
if (['hk', 'tw'].includes(region)) {
|
||||
const categoryMap = await getCategories(region, cache.tryGet);
|
||||
const tag = category ? categoryMap[category].yctMap : null;
|
||||
|
||||
const response = await getArchive(region, limit, tag);
|
||||
const list = parseList(region, response);
|
||||
|
||||
const items = await Promise.all(list.map((item) => parseItem(item, cache.tryGet)));
|
||||
|
||||
return {
|
||||
title: `Yahoo 新聞 ${region.toUpperCase()} - ${category ? categoryMap[category].name : '所有類別'}`,
|
||||
link: `https://${region}.news.yahoo.com/${category ? `${category}/` : ''}archive`,
|
||||
image: 'https://s.yimg.com/cv/apiv2/social/images/yahoo_default_logo-1200x1200.png',
|
||||
item: items,
|
||||
};
|
||||
} else {
|
||||
const rssUrl = `https://${region ? `${region}.` : ''}news.yahoo.com/rss/${category ? `${category}/` : ''}`;
|
||||
const feed = await parser.parseURL(rssUrl);
|
||||
const filteredItems = feed.items.filter((item) => item?.link && !item.link.includes('promotions') && new URL(item.link).hostname.match(/.*\.yahoo\.com$/));
|
||||
const items = await Promise.all(filteredItems.map((item) => parseItem(item, cache.tryGet)));
|
||||
|
||||
return {
|
||||
title: `Yahoo News ${region.toUpperCase()} - ${category ? category.toUpperCase() : 'All'}`,
|
||||
link: feed.link,
|
||||
description: feed.description,
|
||||
item: items,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -4,10 +4,10 @@ import { getProviderList } from './utils';
|
|||
import InvalidParameterError from '@/errors/types/invalid-parameter';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/news/providers/:region',
|
||||
path: '/news/providers/:region/list',
|
||||
categories: ['new-media'],
|
||||
example: '/yahoo/news/providers/tw',
|
||||
parameters: { region: '地區，見上表' },
|
||||
example: '/yahoo/news/providers/tw/list',
|
||||
parameters: { region: '地区, 同路由"新闻来源"中的支持地区, 即 hk 或 tw' },
|
||||
features: {
|
||||
requireConfig: false,
|
||||
requirePuppeteer: false,
|
||||
|
|
@ -16,8 +16,13 @@ export const route: Route = {
|
|||
supportPodcast: false,
|
||||
supportScihub: false,
|
||||
},
|
||||
radar: [
|
||||
{
|
||||
source: ['hk.news.yahoo.com/', 'tw.news.yahoo.com/'],
|
||||
},
|
||||
],
|
||||
name: '新聞來源列表',
|
||||
maintainers: ['TonyRL'],
|
||||
maintainers: ['TonyRL', 'williamgateszhao'],
|
||||
handler,
|
||||
};
|
||||
|
||||
|
|
@ -6,8 +6,8 @@ import InvalidParameterError from '@/errors/types/invalid-parameter';
|
|||
export const route: Route = {
|
||||
path: '/news/provider/:region/:providerId',
|
||||
categories: ['new-media'],
|
||||
example: '/yahoo/news/provider/tw/udn.com',
|
||||
parameters: { region: '地區，見下表', providerId: '新聞來源 ID，可透過下方新聞來源列表獲得' },
|
||||
example: '/yahoo/news/provider/tw/yahoo_tech_tw_942',
|
||||
parameters: { region: '地區, hk 或 tw, 分别表示香港雅虎和台湾雅虎', providerId: '新聞來源 ID, 可透過路由"新聞來源列表"獲得' },
|
||||
features: {
|
||||
requireConfig: false,
|
||||
requirePuppeteer: false,
|
||||
|
|
@ -16,12 +16,27 @@ export const route: Route = {
|
|||
supportPodcast: false,
|
||||
supportScihub: false,
|
||||
},
|
||||
radar: [
|
||||
{
|
||||
source: ['hk.news.yahoo.com/', 'tw.news.yahoo.com/'],
|
||||
},
|
||||
],
|
||||
name: '新聞來源',
|
||||
maintainers: ['TonyRL'],
|
||||
maintainers: ['TonyRL', 'williamgateszhao'],
|
||||
handler,
|
||||
description: `| 香港 | 台灣 |
|
||||
| ---- | ---- |
|
||||
| hk | tw |`,
|
||||
description: `
|
||||
\`Region\`
|
||||
|
||||
| 香港 | 台灣 |
|
||||
| ---- | ---- |
|
||||
| hk | tw |
|
||||
|
||||
\`ProviderId\`
|
||||
|
||||
除了可以通过路由"新聞來源列表"获得外, 也可通过 hk.news.yahoo.com/archive 和 tw.news.yahoo.com/archive 选择"新闻来源"后通过页面 Url 来获得。
|
||||
|
||||
例如 hk.news.yahoo.com/yahoo_movies_hk_660--所有分類/archive, \`yahoo_movies_hk_660\` 就是 ProviderId 。
|
||||
`,
|
||||
};
|
||||
|
||||
async function handler(ctx) {
|
||||
|
|
@ -1,80 +0,0 @@
|
|||
import { Route } from '@/types';
|
||||
import cache from '@/utils/cache';
|
||||
import { getArchive, getCategories, parseList, parseItem } from './utils';
|
||||
import InvalidParameterError from '@/errors/types/invalid-parameter';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/news/:region/:category?',
|
||||
categories: ['new-media'],
|
||||
example: '/yahoo/news/hk/world',
|
||||
parameters: { region: 'Region, see the table below', category: 'Category, see the table below' },
|
||||
features: {
|
||||
requireConfig: false,
|
||||
requirePuppeteer: false,
|
||||
antiCrawler: false,
|
||||
supportBT: false,
|
||||
supportPodcast: false,
|
||||
supportScihub: false,
|
||||
},
|
||||
radar: [
|
||||
{
|
||||
source: ['yahoo.com/'],
|
||||
},
|
||||
],
|
||||
name: 'News',
|
||||
maintainers: ['KeiLongW'],
|
||||
handler,
|
||||
url: 'yahoo.com/',
|
||||
description: `\`Region\`
|
||||
|
||||
| Hong Kong | Taiwan | US |
|
||||
| --------- | ------ | -- |
|
||||
| hk | tw | en |
|
||||
|
||||
<details>
|
||||
<summary>\`Category\` (Hong Kong)</summary>
|
||||
|
||||
| 全部 | 港聞 | 兩岸國際 | 財經 | 娛樂 | 體育 | 健康 | 親子 | 副刊 |
|
||||
| -------- | --------- | -------- | -------- | ------------- | ------ | ------ | --------- | ---------- |
|
||||
| （留空） | hong-kong | world | business | entertainment | sports | health | parenting | supplement |
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary>\`Category\` (Taiwan)</summary>
|
||||
|
||||
| 全部 | 政治 | 財經 | 娛樂 | 運動 | 社會地方 | 國際 | 生活 | 健康 | 科技 | 品味 |
|
||||
| -------- | -------- | ------- | ------------- | ------ | -------- | ----- | --------- | ------ | ---------- | ----- |
|
||||
| （留空） | politics | finance | entertainment | sports | society | world | lifestyle | health | technology | style |
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary>\`Category\` (US)</summary>
|
||||
|
||||
| All | World | Business | Entertainment | Sports | Health |
|
||||
| ------- | ----- | -------- | ------------- | ------ | ------ |
|
||||
| (Empty) | world | business | entertainment | sports | health |
|
||||
</details>`,
|
||||
};
|
||||
|
||||
async function handler(ctx) {
|
||||
const { region, category } = ctx.req.param();
|
||||
if (!['hk', 'tw'].includes(region)) {
|
||||
throw new InvalidParameterError(`Unknown region: ${region}`);
|
||||
}
|
||||
|
||||
const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 20;
|
||||
const categoryMap = await getCategories(region, cache.tryGet);
|
||||
const tag = category ? categoryMap[category].yctMap : null;
|
||||
|
||||
const response = await getArchive(region, limit, tag);
|
||||
const list = parseList(region, response);
|
||||
|
||||
const items = await Promise.all(list.map((item) => parseItem(item, cache.tryGet)));
|
||||
|
||||
return {
|
||||
title: `Yahoo 新聞 - ${category ? categoryMap[category].name : '所有類別'}`,
|
||||
link: `https://${region}.news.yahoo.com/${category ? `${category}/` : ''}archive`,
|
||||
image: 'https://s.yimg.com/cv/apiv2/social/images/yahoo_default_logo-1200x1200.png',
|
||||
item: items,
|
||||
};
|
||||
}
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
import { Route } from '@/types';
|
||||
import cache from '@/utils/cache';
|
||||
import got from '@/utils/got';
|
||||
import parser from '@/utils/rss-parser';
|
||||
import { load } from 'cheerio';
|
||||
import { isValidHost } from '@/utils/valid-host';
|
||||
import InvalidParameterError from '@/errors/types/invalid-parameter';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/news/en/:category?',
|
||||
name: 'Unknown',
|
||||
maintainers: [],
|
||||
handler,
|
||||
};
|
||||
|
||||
async function handler(ctx) {
|
||||
const region = ctx.req.param('region') === 'en' ? '' : ctx.req.param('region').toLowerCase();
|
||||
if (!isValidHost(region) && region !== '') {
|
||||
throw new InvalidParameterError('Invalid region');
|
||||
}
|
||||
const category = ctx.req.param('category') ? ctx.req.param('category').toLowerCase() : '';
|
||||
const rssUrl = `https://${region ? `${region}.` : ''}news.yahoo.com/rss/${category}`;
|
||||
const feed = await parser.parseURL(rssUrl);
|
||||
const filteredItems = feed.items.filter((item) => !item.link.includes('promotions') && new URL(item.link).hostname.match(/.*\.yahoo\.com$/));
|
||||
const items = await Promise.all(
|
||||
filteredItems.map((item) =>
|
||||
cache.tryGet(item.link, async () => {
|
||||
const response = await got({
|
||||
method: 'get',
|
||||
url: item.link,
|
||||
});
|
||||
const $ = load(response.data);
|
||||
const author = `${$('span.caas-author-byline-collapse').text()} @${$('span.caas-attr-provider').text()}`;
|
||||
$('.caas-content-byline-wrapper, .caas-xray-wrapper, .caas-header, .caas-readmore').remove();
|
||||
const description = $('.caas-content-wrapper').html();
|
||||
|
||||
const single = {
|
||||
title: item.title,
|
||||
description,
|
||||
author,
|
||||
pubDate: item.pubDate,
|
||||
link: item.link,
|
||||
};
|
||||
return single;
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
return {
|
||||
title: feed.title,
|
||||
link: feed.link,
|
||||
description: feed.description,
|
||||
item: items,
|
||||
};
|
||||
}
|
||||
|
|
@ -7,7 +7,7 @@ import { parseDate } from '@/utils/parse-date';
|
|||
import path from 'node:path';
|
||||
import { art } from '@/utils/render';
|
||||
|
||||
const getArchive = async (region, limit, tag, providerId) => {
|
||||
const getArchive = async (region, limit, tag, providerId?) => {
|
||||
const { data: response } = await got(
|
||||
`https://${region}.news.yahoo.com/_td-news/api/resource/NCPListService;api=archive;ncpParams=${encodeURIComponent(
|
||||
JSON.stringify({
|
||||
|
|
@ -61,7 +61,7 @@ const getStores = (region, tryGet) =>
|
|||
const appData = JSON.parse(
|
||||
$('script:contains("root.App.main")')
|
||||
.text()
|
||||
.match(/root.App.main\s+=\s+({.+});/)[1]
|
||||
.match(/root.App.main\s+=\s+({.+});/)?.[1] as string
|
||||
);
|
||||
|
||||
return appData.context.dispatcher.stores;
|
||||
|
|
@ -81,41 +81,44 @@ const parseItem = (item, tryGet) =>
|
|||
const $ = load(response);
|
||||
|
||||
const ldJson = JSON.parse($('script[type="application/ld+json"]').first().text());
|
||||
const author = `${$('span.caas-author-byline-collapse').text()} @${$('span.caas-attr-provider').text()}`;
|
||||
const body = $('.caas-body');
|
||||
|
||||
body.find('noscript').remove();
|
||||
// remove padding
|
||||
body.find('.caas-figure-with-pb, .caas-img-container').each((_, ele) => {
|
||||
ele = $(ele);
|
||||
ele.removeAttr('style');
|
||||
const $ele = $(ele);
|
||||
$ele.removeAttr('style');
|
||||
});
|
||||
|
||||
body.find('img').each((_, ele) => {
|
||||
ele = $(ele);
|
||||
let dataSrc = ele.data('src');
|
||||
const $ele = $(ele);
|
||||
let dataSrc = $ele.data('src') as string;
|
||||
|
||||
if (dataSrc) {
|
||||
const match = dataSrc.match(/.*--\/.*--\/(.*)/);
|
||||
if (match?.[1]) {
|
||||
dataSrc = match?.[1];
|
||||
}
|
||||
ele.attr('src', dataSrc);
|
||||
ele.removeAttr('data-src');
|
||||
$ele.attr('src', dataSrc);
|
||||
$ele.removeAttr('data-src');
|
||||
}
|
||||
});
|
||||
// fix blockquote iframe
|
||||
body.find('.caas-iframe').each((_, ele) => {
|
||||
ele = $(ele);
|
||||
if (ele.data('type') === 'youtube') {
|
||||
ele.replaceWith(
|
||||
art(path.join(__dirname, '../../templates/youtube.art'), {
|
||||
id: ele.find('blockquote').data('src').split('/').pop()?.split('?')?.[0],
|
||||
const $ele = $(ele);
|
||||
if ($ele.data('type') === 'youtube') {
|
||||
const blockquoteSrc = $ele.find('blockquote').data('src') as string;
|
||||
$ele.replaceWith(
|
||||
art(path.join(__dirname, '../templates/youtube.art'), {
|
||||
id: blockquoteSrc.split('/').pop()?.split('?')?.[0],
|
||||
})
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
item.description = body.html();
|
||||
item.author = author;
|
||||
item.category = ldJson.keywords;
|
||||
item.updated = parseDate(ldJson.dateModified);
|
||||
|
||||
Loading…
Reference in New Issue