refactor: try to mimic the js cookie behavior and bypass the anti spider in ccdi.gov (#10890)

* try to minic the js cookie behavior and bypass the anti spider

* add more selector for diff article

* regex polished

docs/government.md

@@ -1339,7 +1339,7 @@ pageClass: routes
 
 ### 要闻
 
-<Route author="bigfei" example="/gov/ccdi/yaowenn" path="/gov/ccdi/:path+" :paramsDesc="['路径，默认为 要闻']" anticrawler="1">
+<Route author="bigfei" example="/gov/ccdi/yaowenn" path="/gov/ccdi/:path+" :paramsDesc="['路径，默认为 要闻']">
 
 ::: tip 提示
 

lib/v2/gov/ccdi/index.js

@@ -1,12 +1,5 @@
-/* eslint-disable no-await-in-loop */
 const { rootUrl, parseNewsList, parseArticle } = require('./utils');
 
-const getRandomInt = (min, max) => {
-    min = Math.ceil(min);
-    max = Math.floor(max);
-    return Math.floor(Math.random() * (max - min) + min); // The maximum is exclusive and the minimum is inclusive
-};
-
 module.exports = async (ctx) => {
     const defaultPath = '/yaowenn/';
 
@@ -15,13 +8,7 @@ module.exports = async (ctx) => {
     const currentUrl = `${rootUrl}${pathname}`;
 
     const { list, title } = await parseNewsList(currentUrl, '.list_news_dl li', ctx);
-    const items = [];
-
-    for (const item of list) {
-        items.push(await parseArticle(item, ctx));
-        // sleep randomly for anti rate limit on ccdi site
-        await new Promise((r) => setTimeout(r, getRandomInt(1000, 2500)));
-    }
+    const items = await Promise.all(list.map((item) => parseArticle(item, ctx)));
 
     ctx.state.data = {
         title,

lib/v2/gov/ccdi/utils.js

@@ -8,12 +8,23 @@ const cookieJar = new CookieJar();
 
 const owner = '中央纪委国家监委网站';
 const rootUrl = 'https://www.ccdi.gov.cn';
+const regex = /([A-Z_]+=(?:.*?(?=; max-age)|[a-fA-F0-9]+))/gm;
+
+const parseCookie = async (body) => {
+    const cookies = body.match(regex);
+    if (cookies) {
+        await Promise.all(cookies.map((c) => cookieJar.setCookie(c, rootUrl)));
+    }
+};
 
 const parseNewsList = async (url, selector, ctx) => {
     const response = await got(url, { cookieJar });
-    const $ = cheerio.load(response.data);
+    const data = response.data;
+    parseCookie(data);
+
+    const $ = cheerio.load(data);
     const list = $(selector)
-        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 8)
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 20)
         .toArray()
         .map((item) => {
             item = $(item);
@@ -32,21 +43,23 @@ const parseNewsList = async (url, selector, ctx) => {
 const parseArticle = async (item, ctx) =>
     await ctx.cache.tryGet(item.link, async () => {
         const response = await got(item.link, { cookieJar });
-        const $ = cheerio.load(response.data);
+        const data = response.data;
+        parseCookie(data);
 
-        const title = $('.daty').text().trim();
+        const $ = cheerio.load(data);
+        const title = $('.daty, .source-box').text().trim();
         item.author = title.match(/来源：(.*)发布时间/s)?.[1].trim() ?? owner;
         item.pubDate = timezone(parseDate(title.match(/发布时间：(.*)分享/s)?.[1].trim() ?? item.pubDate), +8);
 
         // Change the img src from relative to absolute for a better compatibility
-        $('.content')
+        $('.content, .bom-box')
             .find('img')
             .each((_, el) => {
                 $(el).attr('src', new URL($(el).attr('src'), item.link).href);
                 // oldsrc is causing freshrss imageproxy not to work correctly
                 $(el).removeAttr('oldsrc').removeAttr('alt');
             });
-        item.description = $('.content').html();
+        item.description = $('.content, .bom-box').html();
         return item;
     });