xiaoxue_admin
/
RSSHub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(route): finally bypass the ccdi.gov cookie and js based anti-spider system (#10944) 

* try to minic the js cookie behavior and bypass the anti spider

* add more selector for diff article

* regex polished

* fix(route): cookie setting should be called in await

* Extract the anti spider from js and finally bypass it
This commit is contained in:
Felix Hsu 2022-09-29 16:37:34 +08:00 committed by GitHub
parent 1fd2a8f273
commit 8edc638f79
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 31 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
lib/v2/gov/ccdi/utils.js
View File

@ -3,18 +3,25 @@ const { parseDate } = require('@/utils/parse-date');
const got = require('@/utils/got');
const timezone = require('@/utils/timezone');
const { CookieJar } = require('tough-cookie');
const { CookieJar, Cookie } = require('tough-cookie');
const cookieJar = new CookieJar();
const owner = '中央纪委国家监委网站';
const rootUrl = 'https://www.ccdi.gov.cn';
const regex = /([A-Z_]+=(?:.*?(?=; max-age)|[a-fA-F0-9]+))/gm;
const regex = /(?<key>[A-Z_]+)=(?<value>(?:.*?(?=; max-age)|[a-fA-F0-9]+))/gm;
const parseCookie = async (body) => {
const cookies = body.match(regex);
if (cookies) {
await Promise.all(cookies.map((c) => cookieJar.setCookie(c, rootUrl)));
let m;
const cookies = [];
while ((m = regex.exec(body)) !== null) {
// This is necessary to avoid infinite loops with zero-width matches
if (m.index === regex.lastIndex) {
regex.lastIndex++;
}
const { key, value } = m.groups;
cookies.push(new Cookie({ key, value }));
}
await Promise.all(cookies.map((c) => cookieJar.setCookie(c, rootUrl)));
};
const parseNewsList = async (url, selector, ctx) => {
@ -40,8 +47,24 @@ const parseNewsList = async (url, selector, ctx) => {
return { list, title };
};
const parseArticle = async (item, ctx) =>
await ctx.cache.tryGet(item.link, async () => {
const changeTrCookie = async () => {
const cookies = await cookieJar.getCookies(rootUrl);
const c = cookies.find((c) => c.key === 'HOY_TR');
if (c) {
const value = c.value;
const tr_array = value.split(',');
const csr = tr_array[0];
const cnv = tr_array[1].split('');
const otr = tr_array[2].split('');
otr[0] = csr.charAt(parseInt(cnv[0], 16));
const nc = new Cookie({ key: 'HOY_TR', value: csr + ',' + cnv.join('') + ',' + otr.join('') + ',0' });
await cookieJar.setCookie(nc, rootUrl);
}
};
const parseArticle = async (item, ctx) => {
await changeTrCookie();
return await ctx.cache.tryGet(item.link, async () => {
const response = await got(item.link, { cookieJar });
const data = response.data;
await parseCookie(data);
@ -62,6 +85,7 @@ const parseArticle = async (item, ctx) =>
item.description = $('.content, .bom-box').html();
return item;
});
};
module.exports = {
rootUrl,