From 64322895eccaf50ca3bd1296ecf174c03741a83e Mon Sep 17 00:00:00 2001
From: Felix Hsu <xuchenfei@gmail.com>
Date: Mon, 26 Sep 2022 19:13:18 +0800
Subject: [PATCH] refactor: try to mimic the js cookie behavior and bypass the
 anti spider in ccdi.gov (#10890)

* try to minic the js cookie behavior and bypass the anti spider

* add more selector for diff article

* regex polished
---
 docs/government.md       |  2 +-
 lib/v2/gov/ccdi/index.js | 15 +--------------
 lib/v2/gov/ccdi/utils.js | 25 +++++++++++++++++++------
 3 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/docs/government.md b/docs/government.md
index 5f7574a17..e54fab715 100644
--- a/docs/government.md
+++ b/docs/government.md
@@ -1339,7 +1339,7 @@ pageClass: routes
 
 ### 要闻
 
-<Route author="bigfei" example="/gov/ccdi/yaowenn" path="/gov/ccdi/:path+" :paramsDesc="['路径，默认为 要闻']" anticrawler="1">
+<Route author="bigfei" example="/gov/ccdi/yaowenn" path="/gov/ccdi/:path+" :paramsDesc="['路径，默认为 要闻']">
 
 ::: tip 提示
 
diff --git a/lib/v2/gov/ccdi/index.js b/lib/v2/gov/ccdi/index.js
index 9ba24aa68..17ee93431 100644
--- a/lib/v2/gov/ccdi/index.js
+++ b/lib/v2/gov/ccdi/index.js
@@ -1,12 +1,5 @@
-/* eslint-disable no-await-in-loop */
 const { rootUrl, parseNewsList, parseArticle } = require('./utils');
 
-const getRandomInt = (min, max) => {
-    min = Math.ceil(min);
-    max = Math.floor(max);
-    return Math.floor(Math.random() * (max - min) + min); // The maximum is exclusive and the minimum is inclusive
-};
-
 module.exports = async (ctx) => {
     const defaultPath = '/yaowenn/';
 
@@ -15,13 +8,7 @@ module.exports = async (ctx) => {
     const currentUrl = `${rootUrl}${pathname}`;
 
     const { list, title } = await parseNewsList(currentUrl, '.list_news_dl li', ctx);
-    const items = [];
-
-    for (const item of list) {
-        items.push(await parseArticle(item, ctx));
-        // sleep randomly for anti rate limit on ccdi site
-        await new Promise((r) => setTimeout(r, getRandomInt(1000, 2500)));
-    }
+    const items = await Promise.all(list.map((item) => parseArticle(item, ctx)));
 
     ctx.state.data = {
         title,
diff --git a/lib/v2/gov/ccdi/utils.js b/lib/v2/gov/ccdi/utils.js
index a6c01b698..caee722d5 100644
--- a/lib/v2/gov/ccdi/utils.js
+++ b/lib/v2/gov/ccdi/utils.js
@@ -8,12 +8,23 @@ const cookieJar = new CookieJar();
 
 const owner = '中央纪委国家监委网站';
 const rootUrl = 'https://www.ccdi.gov.cn';
+const regex = /([A-Z_]+=(?:.*?(?=; max-age)|[a-fA-F0-9]+))/gm;
+
+const parseCookie = async (body) => {
+    const cookies = body.match(regex);
+    if (cookies) {
+        await Promise.all(cookies.map((c) => cookieJar.setCookie(c, rootUrl)));
+    }
+};
 
 const parseNewsList = async (url, selector, ctx) => {
     const response = await got(url, { cookieJar });
-    const $ = cheerio.load(response.data);
+    const data = response.data;
+    parseCookie(data);
+
+    const $ = cheerio.load(data);
     const list = $(selector)
-        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 8)
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 20)
         .toArray()
         .map((item) => {
             item = $(item);
@@ -32,21 +43,23 @@ const parseNewsList = async (url, selector, ctx) => {
 const parseArticle = async (item, ctx) =>
     await ctx.cache.tryGet(item.link, async () => {
         const response = await got(item.link, { cookieJar });
-        const $ = cheerio.load(response.data);
+        const data = response.data;
+        parseCookie(data);
 
-        const title = $('.daty').text().trim();
+        const $ = cheerio.load(data);
+        const title = $('.daty, .source-box').text().trim();
         item.author = title.match(/来源：(.*)发布时间/s)?.[1].trim() ?? owner;
         item.pubDate = timezone(parseDate(title.match(/发布时间：(.*)分享/s)?.[1].trim() ?? item.pubDate), +8);
 
         // Change the img src from relative to absolute for a better compatibility
-        $('.content')
+        $('.content, .bom-box')
             .find('img')
             .each((_, el) => {
                 $(el).attr('src', new URL($(el).attr('src'), item.link).href);
                 // oldsrc is causing freshrss imageproxy not to work correctly
                 $(el).removeAttr('oldsrc').removeAttr('alt');
             });
-        item.description = $('.content').html();
+        item.description = $('.content, .bom-box').html();
         return item;
     });