diff --git a/docs/government.md b/docs/government.md index ea12e9520..8ca07b535 100644 --- a/docs/government.md +++ b/docs/government.md @@ -264,15 +264,17 @@ pageClass: routes ## 国家统计局 -### 统计数据 > 最新发布 +### 通用 - + ::: tip 提示 路径处填写对应页面 URL 中 `http://www.stats.gov.cn/` 后的字段。下面是一个例子。 -若订阅 [统计数据 > 统计标准](http://www.stats.gov.cn/tjsj/tjbz/) 则将对应页面 URL 中 `http://www.stats.gov.cn/` 后的字段 `tjsj/tjbz` 作为路径填入。此时路由为 [`/gov/stats/tjsj/tjbz`](https://rsshub.app/gov/stats/tjsj/tjbz) +若订阅 [数据 > 数据解读](http://www.stats.gov.cn/sj/sjjd/) 则将对应页面 URL 中 `http://www.stats.gov.cn/` 后的字段 `sj/sjjd` 作为路径填入。此时路由为 [`/gov/stats/sj/sjjd`](https://rsshub.app/gov/stats/sj/sjjd) + +若订阅 [新闻 > 时政要闻 > 中央精神](http://www.stats.gov.cn/xw/szyw/zyjs/) 则将对应页面 URL 中 `http://www.stats.gov.cn/` 后的字段 `xw/szyw/zyjs` 作为路径填入。此时路由为 [`/gov/stats/xw/szyw/zyjs`](https://rsshub.app/gov/stats/xw/szyw/zyjs) ::: diff --git a/lib/v2/gov/maintainer.js b/lib/v2/gov/maintainer.js index 918d4b843..9b25e7e41 100644 --- a/lib/v2/gov/maintainer.js +++ b/lib/v2/gov/maintainer.js @@ -40,7 +40,7 @@ module.exports = { '/pbc/tradeAnnouncement': ['nczitzk'], '/pbc/zcyj': ['Fatpandac'], '/sasac/:path+': ['TonyRL'], - '/stats/:path+': ['bigfei'], + '/stats/:path+': ['bigfei', 'nczitzk'], '/zhengce/govall/:advance?': ['ciaranchen'], '/zhengce/wenjian/:pcodeJiguan?': ['ciaranchen'], '/zhengce/zhengceku/:department': ['zxx-457'], diff --git a/lib/v2/gov/radar.js b/lib/v2/gov/radar.js index 916e0cb9c..2e5e027cb 100644 --- a/lib/v2/gov/radar.js +++ b/lib/v2/gov/radar.js @@ -929,8 +929,8 @@ module.exports = { _name: '国家统计局', www: [ { - title: '统计数据 > 最新发布', - docs: 'https://docs.rsshub.app/government.html#guo-jia-tong-ji-ju-tong-ji-shu-ju-zui-xin-fa-bu', + title: '通用', + docs: 'https://docs.rsshub.app/government.html#guo-jia-tong-ji-ju-shu-ju-tong-yong', source: ['/*'], target: (params, url) => `/gov/stats/${new URL(url).href.match(/stats\.gov\.cn\/(.*)/)[1]}`, }, diff --git a/lib/v2/gov/stats/index.js b/lib/v2/gov/stats/index.js index 30e76eca4..25d30c1c3 100644 --- a/lib/v2/gov/stats/index.js +++ b/lib/v2/gov/stats/index.js @@ -1,18 +1,111 @@ -const { parseContList, parseXilan } = require('./utils'); +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const timezone = require('@/utils/timezone'); +const { parseDate } = require('@/utils/parse-date'); +const { art } = require('@/utils/render'); +const path = require('path'); module.exports = async (ctx) => { + const limit = ctx.query.limit ? parseInt(ctx.query.limit) : 15; + + const defaultPath = '/sj/zxfb/'; + const rootUrl = 'http://www.stats.gov.cn'; - const defaultPath = '/tjsj/zxfb/'; + const currentUrl = `${rootUrl}${ctx.path === '/stats' ? defaultPath : ctx.path.replace(/^\/stats(.*)/, '$1/')}`; - let pathname = ctx.path.replace(/(^\/stats|\/$)/g, ''); - pathname = pathname === '' ? defaultPath : pathname.endsWith('/') ? pathname : pathname + '/'; - const currentUrl = `${rootUrl}${pathname}`; + let response = await got({ + method: 'get', + url: rootUrl, + }); - const { list, title } = await parseContList(currentUrl, 'ul.center_list_contlist li a:not([id]), ul.center_list_cont li a:not([id])', ctx); - const items = await Promise.all(list.map((item) => parseXilan(item, ctx))); + const headers = { + cookie: response.headers['set-cookie'].join(' ').match(/(wzws_sessionid=.*?);/)[1], + }; + + response = await got({ + method: 'get', + url: currentUrl, + headers, + }); + + const $ = cheerio.load(response.data); + + let items = $($('a.pchide').length === 0 ? 'a[title]' : '.list-content a.pchide') + .slice(0, limit) + .toArray() + .map((item) => { + item = $(item); + + return { + title: item.attr('title'), + link: new URL(item.attr('href'), currentUrl).href, + }; + }); + + items = await Promise.all( + items.map((item) => + ctx.cache.tryGet(item.link, async () => { + const detailResponse = await got({ + method: 'get', + url: item.link, + headers, + }); + + const content = cheerio.load(detailResponse.data); + + // articles from www.news.cn or www.gov.cn + + if (/(news\.cn|www\.gov\.cn)/.test(item.link)) { + if (content('.year').text()) { + item.pubDate = timezone(parseDate(`${content('.year').text()}/${content('.day').text()} ${content('.time').text()}`, 'YYYY/MM/DD HH:mm:ss'), +8); + item.author = content('.source') + .text() + .replace(/来源：/, '') + .trim(); + } else { + content('.pages_print').remove(); + + const info = content('.info, .pages-date').text().split('来源：'); + item.pubDate = timezone(parseDate(info[0].trim()), +8); + item.author = info.pop(); + } + + item.title = item.title || content('h1').first().text() || content('h2').first().text(); + item.description = content('#detail, .xlcontent, .pages_content').html(); + + return item; + } + + try { + item.author = detailResponse.data.match(/来源：(.*?) { + a = $(a); + return { + link: new URL(a.attr('href'), item.link).href, + name: a.text().trim(), + }; + }), + }); + + return item; + }) + ) + ); ctx.state.data = { - title, + title: $('title').text(), link: currentUrl, item: items, }; diff --git a/lib/v2/gov/stats/templates/attachments.art b/lib/v2/gov/stats/templates/attachments.art deleted file mode 100644 index 65c3e676d..000000000 --- a/lib/v2/gov/stats/templates/attachments.art +++ /dev/null @@ -1,5 +0,0 @@ -{{ each attachments }} -

- {{ $value.text }} -

-{{ /each}} diff --git a/lib/v2/gov/stats/templates/description.art b/lib/v2/gov/stats/templates/description.art new file mode 100644 index 000000000..08297feaf --- /dev/null +++ b/lib/v2/gov/stats/templates/description.art @@ -0,0 +1,12 @@ +{{@ description }} +{{ if attachments }} +
+

附件：

+ +{{ /if }} \ No newline at end of file diff --git a/lib/v2/gov/stats/utils.js b/lib/v2/gov/stats/utils.js deleted file mode 100644 index e4fcbfd03..000000000 --- a/lib/v2/gov/stats/utils.js +++ /dev/null @@ -1,75 +0,0 @@ -const cheerio = require('cheerio'); -const { parseDate } = require('@/utils/parse-date'); -const got = require('@/utils/got'); -const timezone = require('@/utils/timezone'); -const { art } = require('@/utils/render'); -const path = require('path'); - -const parseContList = async (url, selector, ctx) => { - const response = await got(url); - const $ = cheerio.load(response.data); - const list = $(selector) - .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 12) - .toArray() - .map((item) => { - item = $(item); - - const date = item.find('.cont_tit02').text() || item.next('.cont_tit02').text(); - return { - title: item.find('.cont_tit03, .cont_tit01').text(), - link: new URL(item.attr('href'), url).href, - pubDate: parseDate(date, 'YYYY-MM-DD'), - }; - }) - .filter((item) => item.title); // exclude the empty title - const title = $('#PL_DAOHANG') - .text() - .replace(/(.+)首页/, '国家统计局'); - return { list, title }; -}; - -const parseXilan = (item, ctx) => - ctx.cache.tryGet(item.link, async () => { - const response = await got(item.link); - const $ = cheerio.load(response.data); - const title = $('.xilan_titf').text(); - item.author = title.match(/来源：(.*)发布时间/)?.[1].trim() ?? '国家统计局'; - item.pubDate = timezone(parseDate(title.match(/发布时间：(.*)/)?.[1].trim() ?? item.pubDate), +8); - - // Change the img src from relative to absolute for a better compatibility - $('.xilan_con') - .find('img') - .each((_, el) => { - $(el).attr('src', new URL($(el).attr('src'), item.link).href); - // oldsrc is causing freshrss imageproxy not to work correctly - $(el).removeAttr('oldsrc').removeAttr('alt'); - }); - item.description = $('.xilan_con').html(); - - const attachmentTitle = $('.wenzhang_tit').filter(function () { - return $(this).text().trim() === '相关附件'; - }); - if (attachmentTitle.length > 0) { - const attachments = attachmentTitle - .first() - .next('.wenzhang_list') - .find('a') - .toArray() - .map((attachment) => { - attachment = $(attachment); - return { - href: new URL(attachment.attr('href'), item.link).href, - text: attachment.text().trim(), - }; - }); - item.description += art(path.join(__dirname, 'templates/attachments.art'), { - attachments, - }); - } - return item; - }); - -module.exports = { - parseXilan, - parseContList, -};