feat(route): add 国家统计局通用 (#12290)
* chore(deps): bump chrono-node from 2.6.2 to 2.6.3 (#513) Bumps [chrono-node](https://github.com/wanasit/chrono) from 2.6.2 to 2.6.3. - [Release notes](https://github.com/wanasit/chrono/releases) - [Commits](https://github.com/wanasit/chrono/compare/v2.6.2...v2.6.3) --- updated-dependencies: - dependency-name: chrono-node dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps-dev): bump meilisearch from 0.32.2 to 0.32.3 (#518) Bumps [meilisearch](https://github.com/meilisearch/meilisearch-js) from 0.32.2 to 0.32.3. - [Release notes](https://github.com/meilisearch/meilisearch-js/releases) - [Commits](https://github.com/meilisearch/meilisearch-js/compare/v0.32.2...v0.32.3) --- updated-dependencies: - dependency-name: meilisearch dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * feat(route): 国家统计局通用 --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
This commit is contained in:
Ethan Shen
GitHub
parent
8a76f8743b
commit
17e7cbf2cc
|
|
@ -264,15 +264,17 @@ pageClass: routes
|
|||
|
||||
## 国家统计局
|
||||
|
||||
### 统计数据 > 最新发布
|
||||
### 通用
|
||||
|
||||
<Route author="bigfei" example="/gov/stats/tjsj/zxfb" path="/gov/stats/:path+" :paramsDesc="['路径，默认为 统计数据 > 最新发布']">
|
||||
<Route author="bigfei nczitzk" example="/gov/stats/sj/zxfb" path="/gov/stats/:path+" :paramsDesc="['路径，默认为数据最新发布']">
|
||||
|
||||
::: tip 提示
|
||||
|
||||
路径处填写对应页面 URL 中 `http://www.stats.gov.cn/` 后的字段。下面是一个例子。
|
||||
|
||||
若订阅 [统计数据 > 统计标准](http://www.stats.gov.cn/tjsj/tjbz/) 则将对应页面 URL <http://www.stats.gov.cn/tjsj/tjbz/> 中 `http://www.stats.gov.cn/` 后的字段 `tjsj/tjbz` 作为路径填入。此时路由为 [`/gov/stats/tjsj/tjbz`](https://rsshub.app/gov/stats/tjsj/tjbz)
|
||||
若订阅 [数据 > 数据解读](http://www.stats.gov.cn/sj/sjjd/) 则将对应页面 URL <http://www.stats.gov.cn/sj/sjjd/> 中 `http://www.stats.gov.cn/` 后的字段 `sj/sjjd` 作为路径填入。此时路由为 [`/gov/stats/sj/sjjd`](https://rsshub.app/gov/stats/sj/sjjd)
|
||||
|
||||
若订阅 [新闻 > 时政要闻 > 中央精神](http://www.stats.gov.cn/xw/szyw/zyjs/) 则将对应页面 URL <http://www.stats.gov.cn/xw/szyw/zyjs/> 中 `http://www.stats.gov.cn/` 后的字段 `xw/szyw/zyjs` 作为路径填入。此时路由为 [`/gov/stats/xw/szyw/zyjs`](https://rsshub.app/gov/stats/xw/szyw/zyjs)
|
||||
|
||||
:::
|
||||
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ module.exports = {
|
|||
'/pbc/tradeAnnouncement': ['nczitzk'],
|
||||
'/pbc/zcyj': ['Fatpandac'],
|
||||
'/sasac/:path+': ['TonyRL'],
|
||||
'/stats/:path+': ['bigfei'],
|
||||
'/stats/:path+': ['bigfei', 'nczitzk'],
|
||||
'/zhengce/govall/:advance?': ['ciaranchen'],
|
||||
'/zhengce/wenjian/:pcodeJiguan?': ['ciaranchen'],
|
||||
'/zhengce/zhengceku/:department': ['zxx-457'],
|
||||
|
|
|
|||
|
|
@ -929,8 +929,8 @@ module.exports = {
|
|||
_name: '国家统计局',
|
||||
www: [
|
||||
{
|
||||
title: '统计数据 > 最新发布',
|
||||
docs: 'https://docs.rsshub.app/government.html#guo-jia-tong-ji-ju-tong-ji-shu-ju-zui-xin-fa-bu',
|
||||
title: '通用',
|
||||
docs: 'https://docs.rsshub.app/government.html#guo-jia-tong-ji-ju-shu-ju-tong-yong',
|
||||
source: ['/*'],
|
||||
target: (params, url) => `/gov/stats/${new URL(url).href.match(/stats\.gov\.cn\/(.*)/)[1]}`,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1,18 +1,111 @@
|
|||
const { parseContList, parseXilan } = require('./utils');
|
||||
const got = require('@/utils/got');
|
||||
const cheerio = require('cheerio');
|
||||
const timezone = require('@/utils/timezone');
|
||||
const { parseDate } = require('@/utils/parse-date');
|
||||
const { art } = require('@/utils/render');
|
||||
const path = require('path');
|
||||
|
||||
module.exports = async (ctx) => {
|
||||
const limit = ctx.query.limit ? parseInt(ctx.query.limit) : 15;
|
||||
|
||||
const defaultPath = '/sj/zxfb/';
|
||||
|
||||
const rootUrl = 'http://www.stats.gov.cn';
|
||||
const defaultPath = '/tjsj/zxfb/';
|
||||
const currentUrl = `${rootUrl}${ctx.path === '/stats' ? defaultPath : ctx.path.replace(/^\/stats(.*)/, '$1/')}`;
|
||||
|
||||
let pathname = ctx.path.replace(/(^\/stats|\/$)/g, '');
|
||||
pathname = pathname === '' ? defaultPath : pathname.endsWith('/') ? pathname : pathname + '/';
|
||||
const currentUrl = `${rootUrl}${pathname}`;
|
||||
let response = await got({
|
||||
method: 'get',
|
||||
url: rootUrl,
|
||||
});
|
||||
|
||||
const { list, title } = await parseContList(currentUrl, 'ul.center_list_contlist li a:not([id]), ul.center_list_cont li a:not([id])', ctx);
|
||||
const items = await Promise.all(list.map((item) => parseXilan(item, ctx)));
|
||||
const headers = {
|
||||
cookie: response.headers['set-cookie'].join(' ').match(/(wzws_sessionid=.*?);/)[1],
|
||||
};
|
||||
|
||||
response = await got({
|
||||
method: 'get',
|
||||
url: currentUrl,
|
||||
headers,
|
||||
});
|
||||
|
||||
const $ = cheerio.load(response.data);
|
||||
|
||||
let items = $($('a.pchide').length === 0 ? 'a[title]' : '.list-content a.pchide')
|
||||
.slice(0, limit)
|
||||
.toArray()
|
||||
.map((item) => {
|
||||
item = $(item);
|
||||
|
||||
return {
|
||||
title: item.attr('title'),
|
||||
link: new URL(item.attr('href'), currentUrl).href,
|
||||
};
|
||||
});
|
||||
|
||||
items = await Promise.all(
|
||||
items.map((item) =>
|
||||
ctx.cache.tryGet(item.link, async () => {
|
||||
const detailResponse = await got({
|
||||
method: 'get',
|
||||
url: item.link,
|
||||
headers,
|
||||
});
|
||||
|
||||
const content = cheerio.load(detailResponse.data);
|
||||
|
||||
// articles from www.news.cn or www.gov.cn
|
||||
|
||||
if (/(news\.cn|www\.gov\.cn)/.test(item.link)) {
|
||||
if (content('.year').text()) {
|
||||
item.pubDate = timezone(parseDate(`${content('.year').text()}/${content('.day').text()} ${content('.time').text()}`, 'YYYY/MM/DD HH:mm:ss'), +8);
|
||||
item.author = content('.source')
|
||||
.text()
|
||||
.replace(/来源：/, '')
|
||||
.trim();
|
||||
} else {
|
||||
content('.pages_print').remove();
|
||||
|
||||
const info = content('.info, .pages-date').text().split('来源：');
|
||||
item.pubDate = timezone(parseDate(info[0].trim()), +8);
|
||||
item.author = info.pop();
|
||||
}
|
||||
|
||||
item.title = item.title || content('h1').first().text() || content('h2').first().text();
|
||||
item.description = content('#detail, .xlcontent, .pages_content').html();
|
||||
|
||||
return item;
|
||||
}
|
||||
|
||||
try {
|
||||
item.author = detailResponse.data.match(/来源：(.*?)</)[1].trim();
|
||||
} catch (e) {
|
||||
item.author = content('div.detail-title-des h2 span').first().text().split(':').pop().trim();
|
||||
}
|
||||
|
||||
content('.pchide').remove();
|
||||
|
||||
item.title = item.title || content('div.detail-title h1').text();
|
||||
item.pubDate = timezone(parseDate(content('div.detail-title-des h2 p, .info').first().text().trim()), +8);
|
||||
item.description = art(path.join(__dirname, 'templates/description.art'), {
|
||||
description: content('.TRS_Editor').html(),
|
||||
attachments: content('a[oldsrc]')
|
||||
.toArray()
|
||||
.map((a) => {
|
||||
a = $(a);
|
||||
return {
|
||||
link: new URL(a.attr('href'), item.link).href,
|
||||
name: a.text().trim(),
|
||||
};
|
||||
}),
|
||||
});
|
||||
|
||||
return item;
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
ctx.state.data = {
|
||||
title,
|
||||
title: $('title').text(),
|
||||
link: currentUrl,
|
||||
item: items,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,5 +0,0 @@
|
|||
{{ each attachments }}
|
||||
<p>
|
||||
<a href="{{ $value.href }}" >{{ $value.text }}</a>
|
||||
</p>
|
||||
{{ /each}}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
{{@ description }}
|
||||
{{ if attachments }}
|
||||
<br>
|
||||
<p>附件：</p>
|
||||
<ul>
|
||||
{{ each attachments attachment }}
|
||||
<li>
|
||||
<a href="{{ attachment.link }}" >{{ attachment.name }}</a>
|
||||
</li>
|
||||
{{ /each }}
|
||||
</ul>
|
||||
{{ /if }}
|
||||
|
|
@ -1,75 +0,0 @@
|
|||
const cheerio = require('cheerio');
|
||||
const { parseDate } = require('@/utils/parse-date');
|
||||
const got = require('@/utils/got');
|
||||
const timezone = require('@/utils/timezone');
|
||||
const { art } = require('@/utils/render');
|
||||
const path = require('path');
|
||||
|
||||
const parseContList = async (url, selector, ctx) => {
|
||||
const response = await got(url);
|
||||
const $ = cheerio.load(response.data);
|
||||
const list = $(selector)
|
||||
.slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 12)
|
||||
.toArray()
|
||||
.map((item) => {
|
||||
item = $(item);
|
||||
|
||||
const date = item.find('.cont_tit02').text() || item.next('.cont_tit02').text();
|
||||
return {
|
||||
title: item.find('.cont_tit03, .cont_tit01').text(),
|
||||
link: new URL(item.attr('href'), url).href,
|
||||
pubDate: parseDate(date, 'YYYY-MM-DD'),
|
||||
};
|
||||
})
|
||||
.filter((item) => item.title); // exclude the empty title
|
||||
const title = $('#PL_DAOHANG')
|
||||
.text()
|
||||
.replace(/(.+)首页/, '国家统计局');
|
||||
return { list, title };
|
||||
};
|
||||
|
||||
const parseXilan = (item, ctx) =>
|
||||
ctx.cache.tryGet(item.link, async () => {
|
||||
const response = await got(item.link);
|
||||
const $ = cheerio.load(response.data);
|
||||
const title = $('.xilan_titf').text();
|
||||
item.author = title.match(/来源：(.*)发布时间/)?.[1].trim() ?? '国家统计局';
|
||||
item.pubDate = timezone(parseDate(title.match(/发布时间：(.*)/)?.[1].trim() ?? item.pubDate), +8);
|
||||
|
||||
// Change the img src from relative to absolute for a better compatibility
|
||||
$('.xilan_con')
|
||||
.find('img')
|
||||
.each((_, el) => {
|
||||
$(el).attr('src', new URL($(el).attr('src'), item.link).href);
|
||||
// oldsrc is causing freshrss imageproxy not to work correctly
|
||||
$(el).removeAttr('oldsrc').removeAttr('alt');
|
||||
});
|
||||
item.description = $('.xilan_con').html();
|
||||
|
||||
const attachmentTitle = $('.wenzhang_tit').filter(function () {
|
||||
return $(this).text().trim() === '相关附件';
|
||||
});
|
||||
if (attachmentTitle.length > 0) {
|
||||
const attachments = attachmentTitle
|
||||
.first()
|
||||
.next('.wenzhang_list')
|
||||
.find('a')
|
||||
.toArray()
|
||||
.map((attachment) => {
|
||||
attachment = $(attachment);
|
||||
return {
|
||||
href: new URL(attachment.attr('href'), item.link).href,
|
||||
text: attachment.text().trim(),
|
||||
};
|
||||
});
|
||||
item.description += art(path.join(__dirname, 'templates/attachments.art'), {
|
||||
attachments,
|
||||
});
|
||||
}
|
||||
return item;
|
||||
});
|
||||
|
||||
module.exports = {
|
||||
parseXilan,
|
||||
parseContList,
|
||||
};
|
||||
Loading…
Reference in New Issue