xiaoxue_admin
/
RSSHub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(route): /gov/moa/:suburl{.+} (#15012) 

* fix(route): /gov/moa/:suburl{.+}

* fix(route): replace with endsWith & cache.tryGet

* Update lib/routes/gov/moa/moa.ts

Co-authored-by: Tony <TonyRL@users.noreply.github.com>

* Update lib/routes/gov/moa/moa.ts

Co-authored-by: Tony <TonyRL@users.noreply.github.com>

* Update lib/routes/gov/moa/moa.ts

Co-authored-by: Tony <TonyRL@users.noreply.github.com>

* fix: live news broadcast page & govpublic channel

* fix: correct spelling mistakes

---------
This commit is contained in:
lyqluis 2024-04-08 03:18:43 +08:00 committed by GitHub
parent dd58748c7e
commit 8757a93fa9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 121 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

161
lib/routes/gov/moa/moa.ts
View File

@ -5,27 +5,34 @@ import { load } from 'cheerio';
import { parseRelativeDate } from '@/utils/parse-date';
const hostUrl = 'http://www.moa.gov.cn/';
const hostUrlObj = new URL(hostUrl); // 用于在下面判断host
const hostUrlObj = new URL(hostUrl); // 用于在下面判断 host
export const route: Route = {
path: '/moa/:suburl{.+}',
categories: ['government'],
example: '/gov/moa/gk/zcjd/',
radar: [
{
source: ['moa.gov.cn/'],
target: '/moa/:suburl',
},
],
name: 'Unknown',
maintainers: [],
parameters: { suburl: '下级目录，请使用最下级的目录' },
name: '中华人民共和国农业农村部 - 新闻',
maintainers: ['Origami404', 'lyqluis'],
handler,
url: 'moa.gov.cn/',
description: `更多例子：
- \`农业农村部动态\`的网页链接是\`http://www.moa.gov.cn/xw/zwdt/\`, 对应的\`suburl\`是\`xw/zwdt\`
- \`财务公开\`的网页链接是\`http://www.moa.gov.cn/gk/cwgk_1/\`, 对应的\`suburl\`是\`gk/cwgk_1\`
- 像[政策法规](http://www.moa.gov.cn/gk/zcfg/)这种页面(\`http://www.moa.gov.cn/gk/zcfg/\`), 它**不是**一个合法的分类目录，它是\`法律\`, \`行政法规\`, \`部门规章\`等一堆栏目的集合，这时候请点开对应栏目的\`更多 >>\`进入栏目的最下级目录，再根据上面的规则提取\`suburl\`
- 特别地，\`图片新闻\`对应的\`suburl\`为\`xw/tpxw/\`, \`最新公开\`对应的\`suburl\`为\`govpublic\`, \`数据>最新发布\`对应的\`suburl\`为\`sj/zxfb\``,
};
async function handler(ctx) {
const rawSuburl = ctx.req.param('suburl');
const suburl = rawSuburl.slice(-1) === '/' ? rawSuburl : rawSuburl + '/';
// 特殊处理两个, 其他的栏目都可以找到那种一个列表下去的目录
// 特殊处理两个，其他的栏目都可以找到那种一个列表下去的目录
if (suburl === 'xw/tpxw/') {
// 图片新闻
return await dealChannel(suburl, {
@ -34,11 +41,22 @@ async function handler(ctx) {
titleSelector: 'a[class="block w_fill ellipsis adc ahc"]',
dateSelector: 'span',
});
} else if (suburl === 'govpublic/') {
// 公开公告
return await dealChannel('govpublic/1/index.htm', {
} else if (suburl.startsWith('sj/zxfb')) {
// 数据 - 最新发布
return await dealLatestDataChannel();
} else if (suburl.startsWith('gk')) {
// 公开
return await dealChannel(suburl, {
channelTitleSelector: 'title',
listSelector: '.gongkai_centerRList li',
listSelector: '.commonlist li',
titleSelector: 'a',
dateSelector: 'span',
});
} else if (suburl.startsWith('govpublic')) {
// 最新公开
return await dealChannel('govpublic/1/index.htm', {
channelTitleText: '最新公开',
listSelector: '.commonlist li',
titleSelector: 'a',
dateSelector: 'span',
});
@ -52,16 +70,16 @@ async function handler(ctx) {
}
}
// 处理文章列表, 从那里获得一堆要爬取的页面, 然后爬取
// 处理文章列表，从那里获得一堆要爬取的页面，然后爬取
async function dealChannel(suburl, selectors) {
const { channelTitleSelector, listSelector, titleSelector, dateSelector } = selectors;
const { channelTitleSelector, listSelector, titleSelector, dateSelector, channelTitleText } = selectors;
// 为了与下面解析相对链接的dealLink配合, 这里末尾必须保证有一条斜杠
const url = hostUrl + suburl;
const respone = await got.get(url);
const $ = load(respone.data);
// 为了与下面解析相对链接的 dealLink 配合，这里末尾必须保证有一条斜杠
const url = suburl.startsWith('http') ? suburl : hostUrl + suburl;
const response = await got.get(url);
const $ = load(response.data);
const channelTitle = $(channelTitleSelector).text();
const channelTitle = channelTitleText ?? $(channelTitleSelector).text();
const pageInfos = $(listSelector)
.map((i, e) => {
@ -101,8 +119,7 @@ async function dealChannel(suburl, selectors) {
item = await dealGovpublicPage(link, item);
} else {
// 外部文章
item.description = `外部链接: ${item.link}`;
item.author = 'unknown';
item.description = `外部链接：${item.link}`;
}
cache.set(link, JSON.stringify(item));
@ -117,18 +134,30 @@ async function dealChannel(suburl, selectors) {
};
}
// 处理正常文章, 例子: http://www.moa.gov.cn/gk/rsxx_1/202004/t20200421_6342037.htm
// 处理正常文章，例子：http://www.moa.gov.cn/xw/zwdt/202309/t20230915_6436615.htm
async function dealNormalPage(link, item) {
const reponse = await got.get(link);
const $ = load(reponse.data);
const metaElements = $('.bjjMAuthorBox span.dc_3').toArray();
const response = await got.get(link);
const $ = load(response.data);
// 政府网站变动不频繁, 写死第几个应该没有多大关系
// 互动-直播访谈
if (link.includes('zbft')) {
const pageHeader = $('.nybzb').html() ?? '';
const pics = $('.tpsl').html() ?? '';
const content = $('.wzsl').html() ?? '';
item.description = pageHeader + pics + content;
return item;
}
// normal news
const metaElements = $('.bjjMAuthorBox span.dc_2').toArray();
// 政府网站变动不频繁，写死第几个应该没有多大关系
const author = $(metaElements[1]).text();
const source = $(metaElements[2]).text();
item.author = `${author} ${source}`;
// 对于这个网站内的链接, 能提供更精确的时间
// 对于这个网站内的链接，能提供更精确的时间
// 这个的日期跟时间之间的空格数量好像会乱变的
const exactTime = $(metaElements[0]).text();
const dateMatch = /\d{4}-\d{2}-\d{2}/.exec(exactTime);
@ -140,34 +169,86 @@ async function dealNormalPage(link, item) {
return item;
}
// 处理那种带索引号的公示文章, 例子: http://www.moa.gov.cn/govpublic/XZQYJ/202004/t20200420_6341913.htm
// 处理那种带索引号的公示文章，例子：http://www.moa.gov.cn/gk/zcjd/202402/t20240219_6448654.htm
async function dealGovpublicPage(link, item) {
const respone = await got.get(link);
const $ = load(respone.data);
if (item.link.endsWith('.pdf')) {
return item;
}
const response = await got.get(link);
const $ = load(response.data);
const head = $('ul.head');
const body = $('.arc_body');
// 日期时间作者等详细信息被包含在了head里面
// 况且都是政府部门, 提取作者信息无多大意义(还没有特别在页面标注出来), 干脆写在正文
// 而且我也搞不懂到底是发布部门算作者还是写出来公告的部门算还是那个人算...
item.description = head.html() + body.html();
const body = $('.gsj_htmlcon_bot');
const [, year, month, date] = $('.pubtime')
.text()
.match(/：(\d{4})[|年-](\d{1,2})[|月-](\d{1,2})日?/);
const [, author] = $('.pubtime.source')
?.text()
?.match(/：(.+)/) ?? [null, ''];
if (year && month && date) {
item.pubDate = `${year}-${month}-${date}`;
}
item.author = author;
item.description = body.html();
return item;
}
// 处理相对url 和 按链接对文章类型进行分类
async function dealLatestDataChannel() {
const res = await got({
url: 'http://zdscxx.moa.gov.cn:8080/nyb/getMessages',
method: 'post',
json: {
page: 1,
rows: 20,
type: '最新发布',
isLatestMessage: true,
},
});
const items = await Promise.all(
res.data.result.table.map((item) => {
const { date, id } = item;
item.pubDate = date;
const link = (item.link = `http://zdscxx.moa.gov.cn:8080/nyb/pc/messageView.jsp?id=${id}`);
return cache.tryGet(link, async () => {
const { content, source } = await getLatestDataArticleDetail(id);
item.description = content;
item.author = source;
return item;
});
})
);
return {
title: `中华人民共和国农业农村部 - 数据 - 最新发布`,
link: 'http://zdscxx.moa.gov.cn:8080/nyb/pc/messageList.jsp',
item: items,
};
}
async function getLatestDataArticleDetail(id) {
const res = await got({
url: 'http://zdscxx.moa.gov.cn:8080/nyb/getMessagesById',
method: 'post',
form: {
id,
},
});
return res.data.result;
}
// 处理相对 url 和 按链接对文章类型进行分类
function dealLink(element, url) {
const rawLink = element.attr('href');
const { host, href } = new URL(rawLink, url);
// host不同的是外部文章, outside
// url里带govpublic的都是公示文章, govpublic
// 其他的都算普通文章, normal
// host 不同的是外部文章，outside
// url 里带 govpublic 的都是公示文章，govpublic
// 其他的都算普通文章，normal
let pageType = null;
if (host === hostUrlObj.host) {
pageType = href.includes('govpublic') ? 'govpublic' : 'normal';
pageType = href.includes('gk') || href.includes('govpublic') ? 'govpublic' : 'normal';
} else {
pageType = 'outside';
}