Files
RSSHub/lib/routes/gov/moa/moa.ts
T
lyqluisandTony 8757a93fa9 fix(route): /gov/moa/:suburl{.+} (#15012)
* fix(route): /gov/moa/:suburl{.+}

* fix(route): replace with endsWith & cache.tryGet

* Update lib/routes/gov/moa/moa.ts

Co-authored-by: Tony <TonyRL@users.noreply.github.com>

* Update lib/routes/gov/moa/moa.ts

Co-authored-by: Tony <TonyRL@users.noreply.github.com>

* Update lib/routes/gov/moa/moa.ts

Co-authored-by: Tony <TonyRL@users.noreply.github.com>

* fix: live news broadcast page & govpublic channel

* fix: correct spelling mistakes

---------
2024-04-08 03:18:43 +08:00

258 lines
8.9 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { Route } from '@/types';
import cache from '@/utils/cache';
import got from '@/utils/got';
import { load } from 'cheerio';
import { parseRelativeDate } from '@/utils/parse-date';
const hostUrl = 'http://www.moa.gov.cn/';
const hostUrlObj = new URL(hostUrl); // 用于在下面判断 host
export const route: Route = {
path: '/moa/:suburl{.+}',
categories: ['government'],
example: '/gov/moa/gk/zcjd/',
radar: [
{
source: ['moa.gov.cn/'],
target: '/moa/:suburl',
},
],
parameters: { suburl: '下级目录,请使用最下级的目录' },
name: '中华人民共和国农业农村部 - 新闻',
maintainers: ['Origami404', 'lyqluis'],
handler,
url: 'moa.gov.cn/',
description: `更多例子:
- \`农业农村部动态\`的网页链接是\`http://www.moa.gov.cn/xw/zwdt/\`, 对应的\`suburl\`是\`xw/zwdt\`
- \`财务公开\`的网页链接是\`http://www.moa.gov.cn/gk/cwgk_1/\`, 对应的\`suburl\`是\`gk/cwgk_1\`
- 像[政策法规](http://www.moa.gov.cn/gk/zcfg/)这种页面(\`http://www.moa.gov.cn/gk/zcfg/\`), 它**不是**一个合法的分类目录,它是\`法律\`, \`行政法规\`, \`部门规章\`等一堆栏目的集合,这时候请点开对应栏目的\`更多 >>\`进入栏目的最下级目录,再根据上面的规则提取\`suburl\`
- 特别地,\`图片新闻\`对应的\`suburl\`为\`xw/tpxw/\`, \`最新公开\`对应的\`suburl\`为\`govpublic\`, \`数据>最新发布\`对应的\`suburl\`为\`sj/zxfb\``,
};
async function handler(ctx) {
const rawSuburl = ctx.req.param('suburl');
const suburl = rawSuburl.slice(-1) === '/' ? rawSuburl : rawSuburl + '/';
// 特殊处理两个,其他的栏目都可以找到那种一个列表下去的目录
if (suburl === 'xw/tpxw/') {
// 图片新闻
return await dealChannel(suburl, {
channelTitleSelector: '.pub-media2-head',
listSelector: '.tupian_list li',
titleSelector: 'a[class="block w_fill ellipsis adc ahc"]',
dateSelector: 'span',
});
} else if (suburl.startsWith('sj/zxfb')) {
// 数据 - 最新发布
return await dealLatestDataChannel();
} else if (suburl.startsWith('gk')) {
// 公开
return await dealChannel(suburl, {
channelTitleSelector: 'title',
listSelector: '.commonlist li',
titleSelector: 'a',
dateSelector: 'span',
});
} else if (suburl.startsWith('govpublic')) {
// 最新公开
return await dealChannel('govpublic/1/index.htm', {
channelTitleText: '最新公开',
listSelector: '.commonlist li',
titleSelector: 'a',
dateSelector: 'span',
});
} else {
return await dealChannel(suburl, {
channelTitleSelector: '.pub-media1-head-title',
listSelector: '.ztlb',
titleSelector: 'a',
dateSelector: 'span',
});
}
}
// 处理文章列表,从那里获得一堆要爬取的页面,然后爬取
async function dealChannel(suburl, selectors) {
const { channelTitleSelector, listSelector, titleSelector, dateSelector, channelTitleText } = selectors;
// 为了与下面解析相对链接的 dealLink 配合,这里末尾必须保证有一条斜杠
const url = suburl.startsWith('http') ? suburl : hostUrl + suburl;
const response = await got.get(url);
const $ = load(response.data);
const channelTitle = channelTitleText ?? $(channelTitleSelector).text();
const pageInfos = $(listSelector)
.map((i, e) => {
const element = $(e);
const titleElement = element.find(titleSelector);
const title = titleElement.text();
const [link, pageType] = dealLink(titleElement, url);
const dateraw = element.find(dateSelector).text().trim();
return {
pageType,
title,
link,
// 先在这里获取一个精确到日的时间
// 如果是正常文章的话可以在那里提取更精确的时间
// 如果是公示文章或者站外文章的话只能用这个保底了
pubDate: parseRelativeDate(dateraw),
};
})
.get();
const items = await Promise.all(
pageInfos.map(async (item) => {
const link = item.link;
const cacheIn = await cache.get(link);
if (cacheIn) {
return JSON.parse(cacheIn);
}
if (item.pageType === 'normal') {
// 正常文章
item = await dealNormalPage(link, item);
} else if (item.pageType === 'govpublic') {
// 公示文章
item = await dealGovpublicPage(link, item);
} else {
// 外部文章
item.description = `外部链接:${item.link}`;
}
cache.set(link, JSON.stringify(item));
return item;
})
);
return {
title: `中华人民共和国农业农村部 - ${channelTitle}`,
link: url,
item: items,
};
}
// 处理正常文章,例子:http://www.moa.gov.cn/xw/zwdt/202309/t20230915_6436615.htm
async function dealNormalPage(link, item) {
const response = await got.get(link);
const $ = load(response.data);
// 互动-直播访谈
if (link.includes('zbft')) {
const pageHeader = $('.nybzb').html() ?? '';
const pics = $('.tpsl').html() ?? '';
const content = $('.wzsl').html() ?? '';
item.description = pageHeader + pics + content;
return item;
}
// normal news
const metaElements = $('.bjjMAuthorBox span.dc_2').toArray();
// 政府网站变动不频繁,写死第几个应该没有多大关系
const author = $(metaElements[1]).text();
const source = $(metaElements[2]).text();
item.author = `${author} ${source}`;
// 对于这个网站内的链接,能提供更精确的时间
// 这个的日期跟时间之间的空格数量好像会乱变的
const exactTime = $(metaElements[0]).text();
const dateMatch = /\d{4}-\d{2}-\d{2}/.exec(exactTime);
const timeMatch = /\d{2}:\d{2}/.exec(exactTime);
item.pubDate = parseRelativeDate(`${dateMatch[0]} ${timeMatch[0]}`);
item.description = $('.arc_body').html();
return item;
}
// 处理那种带索引号的公示文章,例子:http://www.moa.gov.cn/gk/zcjd/202402/t20240219_6448654.htm
async function dealGovpublicPage(link, item) {
if (item.link.endsWith('.pdf')) {
return item;
}
const response = await got.get(link);
const $ = load(response.data);
const body = $('.gsj_htmlcon_bot');
const [, year, month, date] = $('.pubtime')
.text()
.match(/:(\d{4})[|年-](\d{1,2})[|月-](\d{1,2})日?/);
const [, author] = $('.pubtime.source')
?.text()
?.match(/:(.+)/) ?? [null, ''];
if (year && month && date) {
item.pubDate = `${year}-${month}-${date}`;
}
item.author = author;
item.description = body.html();
return item;
}
async function dealLatestDataChannel() {
const res = await got({
url: 'http://zdscxx.moa.gov.cn:8080/nyb/getMessages',
method: 'post',
json: {
page: 1,
rows: 20,
type: '最新发布',
isLatestMessage: true,
},
});
const items = await Promise.all(
res.data.result.table.map((item) => {
const { date, id } = item;
item.pubDate = date;
const link = (item.link = `http://zdscxx.moa.gov.cn:8080/nyb/pc/messageView.jsp?id=${id}`);
return cache.tryGet(link, async () => {
const { content, source } = await getLatestDataArticleDetail(id);
item.description = content;
item.author = source;
return item;
});
})
);
return {
title: `中华人民共和国农业农村部 - 数据 - 最新发布`,
link: 'http://zdscxx.moa.gov.cn:8080/nyb/pc/messageList.jsp',
item: items,
};
}
async function getLatestDataArticleDetail(id) {
const res = await got({
url: 'http://zdscxx.moa.gov.cn:8080/nyb/getMessagesById',
method: 'post',
form: {
id,
},
});
return res.data.result;
}
// 处理相对 url 和 按链接对文章类型进行分类
function dealLink(element, url) {
const rawLink = element.attr('href');
const { host, href } = new URL(rawLink, url);
// host 不同的是外部文章,outside
// url 里带 govpublic 的都是公示文章,govpublic
// 其他的都算普通文章,normal
let pageType = null;
if (host === hostUrlObj.host) {
pageType = href.includes('gk') || href.includes('govpublic') ? 'govpublic' : 'normal';
} else {
pageType = 'outside';
}
return [href, pageType];
}