diff --git a/docs/README.md b/docs/README.md index 4d45e2429..ccea88eb7 100755 --- a/docs/README.md +++ b/docs/README.md @@ -2804,6 +2804,30 @@ Example: `https://store.steampowered.com/search/?specials=1&term=atelier` 中的 +#### 江苏省 + + + +| 省政府常务会议 | 要闻关注 | 部门资讯 | 市县动态 | 政策解读 | +| :---------------: | :------------: | :--------: | :---------: | :-------------------: | +| executive-meeting | important-news | department | city-county | policy-interpretation | + +| 政府信息公开年度报告 | 政府信息公开制度 | 省政府及办公厅文件 | 规范性文件 | +| :------------------: | :-------------------: | :----------------: | :----------------: | +| annual-report | information-publicity | documentation | normative-document | + +| 立法意见征集 | 意见征集 | +| :----------------------------: | :----------------: | +| legislative-opinion-collection | opinion-collection | + +##### 南京市 + + + +| 南京信息 | 部门动态 | 各区动态 | 民生信息 | +| :------: | :--------: | :------: | :--------: | +| news | department | district | livelihood | + ### 中华人民共和国生态环境部 diff --git a/lib/router.js b/lib/router.js index 6501ad57e..9762f251c 100755 --- a/lib/router.js +++ b/lib/router.js @@ -1034,6 +1034,8 @@ router.get('/cyzone/author/:id', require('./routes/cyzone/author')); // 政府 router.get('/gov/zhengce/zuixin', require('./routes/gov/zhengce/zuixin')); router.get('/gov/zhengce/wenjian/:pcodeJiguan?', require('./routes/gov/zhengce/wenjian')); +router.get('/gov/province/:name/:category', require('./routes/gov/province')); +router.get('/gov/city/:name/:category', require('./routes/gov/city')); // 中华人民共和国生态环境部 router.get('/gov/mee/gs', require('./routes/gov/mee/gs')); diff --git a/lib/routes/gov/city/index.js b/lib/routes/gov/city/index.js new file mode 100644 index 000000000..577e95ae8 --- /dev/null +++ b/lib/routes/gov/city/index.js @@ -0,0 +1,26 @@ +module.exports = async (ctx) => { + const { name, category } = ctx.params; + let url = ''; + + switch (name) { + case 'nanjing': + url = 'http://www.nanjing.gov.cn'; + break; + default: + console.log('URL pattern not matched'); + } + + if (url === '') { + ctx.throw(404, 'Cannot find page'); + return; + } + + try { + const getRSS = require(`./${name}`); + const responseData = await getRSS(url, category); + ctx.state.data = responseData; + } catch (error) { + console.error(error); + ctx.throw(404, 'Cannot find page'); + } +}; diff --git a/lib/routes/gov/city/nanjing/getContent.js b/lib/routes/gov/city/nanjing/getContent.js new file mode 100644 index 000000000..a898cd34f --- /dev/null +++ b/lib/routes/gov/city/nanjing/getContent.js @@ -0,0 +1,68 @@ +const axios = require('../../../../utils/axios'); +const cheerio = require('cheerio'); + +module.exports = async (link, totalPage = 1) => { + let title = ''; + let description = ''; + let item = []; + try { + const result = await extract(link); + title = result.title; + description = result.description; + item = result.item; + + const promises = []; + for (let page = 2; page <= totalPage; page++) { + promises.push(extract(link, page)); + } + const pageContent = await Promise.all(promises); + for (const content of pageContent) { + if (Array.isArray(content)) { + item = item.concat(content); + } + } + } catch (err) { + console.error('Cannot load page content'); + } + + return { title, link, description, item }; +}; + +async function extract(url, page = 1) { + let pageLink = `${url}index.html`; + if (page !== 1) { + pageLink = `${url}index_${page - 1}.html`; + } + + const response = await axios.get(pageLink); + + const data = response.data; + const $ = cheerio.load(data); + const list = $('.center ul li'); + + const resultItem = list.map((index, item) => { + const $item = $(item); + const $aTag = $item.find('a'); + + const title = $aTag.text(); + const description = $aTag.attr('title'); + let pubDate = $item.find('span').text(); + pubDate = new Date(pubDate).toUTCString(); + let link = $aTag.attr('href'); + link = `${url}${link.substring(2)}`; + + return { title, description, pubDate, link }; + }); + + if (page === 1) { + const name = $('head title'); + const category = $('.zxft_title span').text(); + return { + title: category, + description: `${category} - ${name}`, + item: resultItem.get(), + }; + } + + return resultItem.get(); +} diff --git a/lib/routes/gov/city/nanjing/index.js b/lib/routes/gov/city/nanjing/index.js new file mode 100644 index 000000000..a8c7eb50d --- /dev/null +++ b/lib/routes/gov/city/nanjing/index.js @@ -0,0 +1,31 @@ +const getContent = require('./getContent'); + +const TOTAL_PAGE = 3; + +module.exports = async (homePage, category) => { + let url = ''; + + switch (category) { + case 'news': + url = `${homePage}/njxx/`; + break; + case 'department': + url = `${homePage}/bmdt/`; + break; + case 'district': + url = `${homePage}/gqdt/`; + break; + case 'livelihood': + url = `${homePage}/mszx/`; + break; + default: + console.log('URL pattern not matched'); + } + + if (url === '') { + throw new Error('Nanjing, Cannot find page'); + } + + const responseData = await getContent(url, TOTAL_PAGE); + return responseData; +}; diff --git a/lib/routes/gov/province/index.js b/lib/routes/gov/province/index.js new file mode 100644 index 000000000..a35bba135 --- /dev/null +++ b/lib/routes/gov/province/index.js @@ -0,0 +1,26 @@ +module.exports = async (ctx) => { + const { name, category } = ctx.params; + let url = ''; + + switch (name) { + case 'jiangsu': + url = 'http://www.jiangsu.gov.cn'; + break; + default: + console.log('URL pattern not matched'); + } + + if (url === '') { + ctx.throw(404, 'Cannot find page'); + return; + } + + try { + const getRSS = require(`./${name}`); + const responseData = await getRSS(url, category); + ctx.state.data = responseData; + } catch (error) { + console.error(error); + ctx.throw(404, 'Cannot find page'); + } +}; diff --git a/lib/routes/gov/province/jiangsu/getContent.js b/lib/routes/gov/province/jiangsu/getContent.js new file mode 100644 index 000000000..3de946484 --- /dev/null +++ b/lib/routes/gov/province/jiangsu/getContent.js @@ -0,0 +1,79 @@ +const axios = require('../../../../utils/axios'); +const cheerio = require('cheerio'); + +const urlTemplate = (homePage, cid, uid) => { + let template = `${homePage}/col/col${cid}/index.html`; + if (uid) { + template += `?uid=${uid}&pageNum=`; + } + return template; +}; + +module.exports = async (homePage, cid, uid, totalPage = 1) => { + let title = ''; + let description = ''; + let item = []; + try { + const result = await extract(homePage, cid, uid); + title = result.title; + description = result.description; + item = result.item; + + const promises = []; + for (let page = 2; page <= totalPage; page++) { + promises.push(extract(homePage, cid, uid, page)); + } + const pageContent = await Promise.all(promises); + for (const content of pageContent) { + if (Array.isArray(content)) { + item = item.concat(content); + } + } + } catch (err) { + console.error('Cannot load page content'); + } + + const link = urlTemplate(homePage, cid); + + return { title, link, description, item }; +}; + +async function extract(homePage, cid, uid, page = 1) { + const pageLink = urlTemplate(homePage, cid, uid); + + const response = await axios.get(`${pageLink}${page}`); + + const data = response.data; + const $ = cheerio.load(data); + const list = $(`#${uid}`) + .html() + .match(/
  • .+<\/li>/g); + + const resultItem = list.map((item) => { + const $item = cheerio.load(item)('li'); + const $aTag = $item.find('a'); + + const title = $aTag.text(); + const description = $aTag.attr('title'); + let pubDate = $item.find('span').text(); + pubDate = new Date(pubDate).toUTCString(); + let link = $aTag.attr('href'); + link = `${homePage}${link}`; + + return { title, description, pubDate, link }; + }); + + if (page === 1) { + const title = $('head title'); + const [name, category] = $(title) + .text() + .split(' '); + return { + title: category, + description: `${category} - ${name}`, + item: resultItem, + }; + } + + return resultItem; +} diff --git a/lib/routes/gov/province/jiangsu/index.js b/lib/routes/gov/province/jiangsu/index.js new file mode 100644 index 000000000..2c3d4cf04 --- /dev/null +++ b/lib/routes/gov/province/jiangsu/index.js @@ -0,0 +1,64 @@ +const getContent = require('./getContent'); + +const TOTAL_PAGE = 1; + +module.exports = async (homePage, category) => { + let cid = ''; + let uid = ''; + + switch (category) { + case 'executive-meeting': + cid = 33716; + uid = 265624; + break; + case 'important-news': + cid = 60096; + uid = 212860; + break; + case 'department': + cid = 60085; + uid = 212860; + break; + case 'city-county': + cid = 33718; + uid = 212860; + break; + case 'documentation': + cid = 32646; + uid = 265638; + break; + case 'policy-interpretation': + cid = 32648; + uid = 158542; + break; + case 'normative-document': + cid = 66109; + uid = 158542; + break; + case 'legislative-opinion-collection': + cid = 69933; + uid = 158542; + break; + case 'opinion-collection': + cid = 31344; + uid = 158542; + break; + case 'annual-report': + cid = 31251; + uid = 158542; + break; + case 'information-publicity': + cid = 31319; + uid = 158542; + break; + default: + console.log('URL pattern not matched'); + } + + if (cid === '') { + throw new Error('Jiangsu, Cannot find page'); + } + + const responseData = await getContent(homePage, cid, uid, TOTAL_PAGE); + return responseData; +};