fix(route): XSIJISHE need cloudflare (#16118)

* 1. 修复sehuatang 403

* 修复sehuatang主页 403

* 过滤不必要的请求

* 修复XSIJISHE需要登录

* 修复XSIJISHE rank需要登录

* This does not require cookie.url

* 修复可能导致的【抓取原帖失败】问题

* Close the page right after getting page content.

* fix headers is not defined

* xsijishe bypass cloudflare

lib/config.ts

@@ -289,6 +289,7 @@ export type Config = {
     };
     xsijishe: {
         cookie?: string;
+        userAgent?: string;
     };
     xueqiu: {
         cookies?: string;
@@ -656,6 +657,7 @@ const calculateValue = () => {
         },
         xsijishe: {
             cookie: envs.XSIJISHE_COOKIE,
+            user_agent: envs.XSIJISHE_USER_AGENT,
         },
         xueqiu: {
             cookies: envs.XUEQIU_COOKIES,

lib/routes/xsijishe/forum.ts

@@ -17,6 +17,10 @@ export const route: Route = {
                 name: 'XSIJISHE_COOKIE',
                 description: '',
             },
+            {
+                name: 'XSIJISHE_USER_AGENT',
+                description: '',
+            },
         ],
         requirePuppeteer: false,
         antiCrawler: false,
@@ -39,6 +43,7 @@ async function handler(ctx) {
         'Accept-Encoding': 'gzip, deflate, br',
         'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
         Cookie: config.xsijishe.cookie,
+        'User-Agent': config.xsijishe.userAgent,
     };
     const resp = await got(url, {
         headers,

lib/routes/xsijishe/rank.ts

@@ -3,6 +3,7 @@ import { Route } from '@/types';
 import cache from '@/utils/cache';
 import got from '@/utils/got';
 import { load } from 'cheerio';
+import { config } from '@/config';
 const baseUrl = 'https://xsijishe.com';
 
 export const route: Route = {
@@ -11,7 +12,16 @@ export const route: Route = {
     example: '/xsijishe/rank/weekly',
     parameters: { type: '排行榜类型: weekly | monthly' },
     features: {
-        requireConfig: false,
+        requireConfig: [
+            {
+                name: 'XSIJISHE_COOKIE',
+                description: '',
+            },
+            {
+                name: 'XSIJISHE_USER_AGENT',
+                description: '',
+            },
+        ],
         requirePuppeteer: false,
         antiCrawler: false,
         supportBT: false,
@@ -37,7 +47,15 @@ async function handler(ctx) {
         throw new InvalidParameterError('Invalid rank type');
     }
     const url = `${baseUrl}/portal.php`;
-    const resp = await got(url);
+    const headers = {
+        'Accept-Encoding': 'gzip, deflate, br',
+        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
+        Cookie: config.xsijishe.cookie,
+        'User-Agent': config.xsijishe.user_agent,
+    };
+    const resp = await got(url, {
+        headers,
+    });
     const $ = load(resp.data);
     let items = $(`#${rankId} dd`)
         .toArray()
@@ -53,7 +71,9 @@ async function handler(ctx) {
     items = await Promise.all(
         items.map((item) =>
             cache.tryGet(item.link, async () => {
-                const resp = await got(item.link);
+                const resp = await got(item.link, {
+                    headers,
+                });
                 const $ = load(resp.data);
                 const firstViewBox = $('.t_f').first();