xiaoxue_admin
/
RSSHub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(routes): remove hard-coded UA and use header presets (#21559)

This commit is contained in:
Tony 2026-03-31 00:40:26 +08:00 committed by GitHub
parent 795028ca09
commit 7d2baa89e5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 29 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lib/routes/changba/user.tsx
View File

@ -5,8 +5,7 @@ import type { Route } from '@/types';
import { ViewType } from '@/types';
import cache from '@/utils/cache';
import got from '@/utils/got';
const headers = { 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1' };
import { PRESETS } from '@/utils/header-generator';
export const route: Route = {
path: '/:userid',
@ -38,7 +37,7 @@ async function handler(ctx) {
const response = await got({
method: 'get',
url,
headers,
headerGeneratorOptions: PRESETS.MODERN_IOS,
});
const data = response.data;
const $ = load(data);
@ -54,7 +53,7 @@ async function handler(ctx) {
const result = await got({
method: 'get',
url: link,
headers,
headerGeneratorOptions: PRESETS.MODERN_IOS,
});
const re = /workid: '\d+'/;

7
lib/routes/ddosi/category.ts
View File

@ -1,8 +1,8 @@
import { load } from 'cheerio';
import { config } from '@/config';
import type { Route } from '@/types';
import got from '@/utils/got';
import { PRESETS } from '@/utils/header-generator';
import { parseDate } from '@/utils/parse-date';
export const route: Route = {
@ -25,22 +25,21 @@ export const route: Route = {
},
],
name: '分类',
maintainers: [],
maintainers: ['XinRoom'],
handler,
url: 'ddosi.org/',
};
async function handler(ctx) {
const url = 'https://www.ddosi.org/category';
const userAgent = config.ua || 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1';
const category = ctx.req.param('category');
const response = await got({
method: 'get',
url: `${url}/${category}/`,
headers: {
'User-Agent': userAgent,
Referer: url,
},
headerGeneratorOptions: PRESETS.MODERN_IOS,
});
const $ = load(response.data);
const list = $('main>article').toArray();

8
lib/routes/ddosi/index.ts
View File

@ -1,19 +1,20 @@
import { load } from 'cheerio';
import { config } from '@/config';
import type { Route } from '@/types';
import got from '@/utils/got';
import { PRESETS } from '@/utils/header-generator';
import { parseDate } from '@/utils/parse-date';
export const route: Route = {
path: '/',
example: '/ddosi',
radar: [
{
source: ['ddosi.org/'],
target: '',
},
],
name: 'Unknown',
name: '首页',
maintainers: ['XinRoom'],
handler,
url: 'ddosi.org/',
@ -21,14 +22,13 @@ export const route: Route = {
async function handler() {
const url = 'https://www.ddosi.org/';
const userAgent = config.ua || 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1';
const response = await got({
method: 'get',
url: String(url),
headers: {
'User-Agent': userAgent,
Referer: url,
},
headerGeneratorOptions: PRESETS.MODERN_IOS,
});
const $ = load(response.data);
const list = $('main>article').toArray();

5
lib/routes/dianping/user.ts
View File

@ -1,5 +1,6 @@
import { config } from '@/config';
import type { Route } from '@/types';
import { PRESETS } from '@/utils/header-generator';
import ofetch from '@/utils/ofetch';
export const route: Route = {
@ -58,12 +59,10 @@ const starMap: Record<number, string> = {
async function handler(ctx) {
const id = ctx.req.param('id');
const userAgent = 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1';
const userPage = `https://m.dianping.com/userprofile/${id}`;
const cookie = config.dianping.cookie;
const headers: Record<string, string> = {
'User-Agent': userAgent,
Referer: userPage,
};
@ -73,6 +72,7 @@ async function handler(ctx) {
const pageResponse = await ofetch(userPage, {
headers,
headerGeneratorOptions: PRESETS.MODERN_IOS,
});
const nickNameReg = /window\.nickName = "(.*?)"/g;
@ -80,6 +80,7 @@ async function handler(ctx) {
const response = await ofetch(`https://m.dianping.com/member/ajax/NobleUserFeeds?userId=${id}`, {
headers,
headerGeneratorOptions: PRESETS.MODERN_IOS,
});
const data = response.data;

1
lib/routes/hyperdash/utils.ts
View File

@ -21,7 +21,6 @@ export async function fetchTopTraders(): Promise<TraderData[]> {
const traders: TraderData[] = await ofetch(apiUrl, {
headers: {
'x-api-key': 'hyperdash_public_7vN3mK8pQ4wX2cL9hF5tR1bY6gS0jD',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
},
});

7
lib/routes/jandan/utils.ts
View File

@ -4,15 +4,12 @@ import type { DataItem } from '@/types';
import ofetch from '@/utils/ofetch';
import { parseDate } from '@/utils/parse-date';
export const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36';
/**
* Extract page ID from script tags in HTML
*/
export const extractPageId = async (url: string, referer: string): Promise<string> => {
const response = await ofetch(url, {
headers: {
'User-Agent': USER_AGENT,
Referer: referer,
Accept: 'application/json, text/plain, */*',
},
@ -39,7 +36,6 @@ export const handleTopSection = async (rootUrl: string, type: string): Promise<{
const apiUrl = `${rootUrl}/api/top/${type}`;
const response = await ofetch(apiUrl, {
headers: {
'User-Agent': USER_AGENT,
Referer: rootUrl,
Accept: 'application/json, text/plain, */*',
},
@ -114,7 +110,6 @@ export const handleForumSection = async (rootUrl: string): Promise<{ title: stri
const apiUrl = `${rootUrl}/api/forum/posts/${forumId}?page=1`;
const forumData = await ofetch(apiUrl, {
headers: {
'User-Agent': USER_AGENT,
Referer: currentUrl,
Accept: 'application/json, text/plain, */*',
},
@ -174,7 +169,6 @@ export const handleCommentSection = async (rootUrl: string, category: string): P
const response = await ofetch(currentUrl, {
headers: {
'User-Agent': USER_AGENT,
Referer: rootUrl,
Accept: 'application/json, text/plain, */*',
},
@ -200,7 +194,6 @@ export const handleCommentSection = async (rootUrl: string, category: string): P
const apiUrl = `${rootUrl}/api/comment/post/${pageId}?order=desc&page=1`;
const commentsData = await ofetch(apiUrl, {
headers: {
'User-Agent': USER_AGENT,
Referer: currentUrl,
Accept: 'application/json, text/plain, */*',
},

1
lib/routes/kuaidi100/utils.ts
View File

@ -230,7 +230,6 @@ export default {
headers: {
Referer: 'https://www.kuaidi100.com/',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,ja;q=0.7',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36',
Cookie: `${cookie.globacsrftoken}; ${cookie.csrf}; ${cookie.wwwid}; ${cookie.dasddocHref}; ${cookie.dasddocReferrer}; ${
cookie.dasddocTitl
}; addcom=${number}; addnu=${id}; snt_query_meta=${queryMeta}; sortStatus=0; Hm_lpvt_22ea01af58ba2be0fec7c11b25e88e6c=${timestamp}; Hm_lvt_22ea01af58ba2be0fec7c11b25e88e6c=${timestamp - 1642}`,

4
lib/routes/qiche365/recall.ts
View File

@ -7,8 +7,6 @@ import timezone from '@/utils/timezone';
const baseUrl = 'https://www.qiche365.org.cn';
const userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
export const route: Route = {
path: '/recall/:channel',
name: '汽车召回',
@ -34,7 +32,6 @@ async function handler(ctx): Promise<Data> {
// second request with those cookies returns the actual JSON data.
const initResponse = await ofetch.raw(targetUrl, {
headers: {
'User-Agent': userAgent,
'Accept-Language': 'zh-CN,zh;q=0.9',
},
ignoreResponseError: true,
@ -44,7 +41,6 @@ async function handler(ctx): Promise<Data> {
const { html } = await ofetch(targetUrl, {
headers: {
'User-Agent': userAgent,
'Accept-Language': 'zh-CN,zh;q=0.9',
Cookie: cookies,
},

7
lib/routes/qidian/author.tsx
View File

@ -2,10 +2,7 @@ import { load } from 'cheerio';
import type { Route } from '@/types';
import got from '@/utils/got';
const headers = {
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Mobile/15E148 Safari/604.1',
};
import { PRESETS } from '@/utils/header-generator';
export const route: Route = {
path: '/author/:id',
@ -35,7 +32,7 @@ async function handler(ctx) {
const currentUrl = `https://my.qidian.com/author/${id}/`;
// Reason: PC site (my.qidian.com) returns anti-bot JS challenge; mobile site has SSR data
const response = await got(`https://m.qidian.com/author/${id}/`, { headers });
const response = await got(`https://m.qidian.com/author/${id}/`, { headerGeneratorOptions: PRESETS.MODERN_IOS });
const $ = load(response.data);
const { pageContext } = JSON.parse($('#vite-plugin-ssr_pageContext').text());
const pageData = pageContext.pageProps.pageData;

10
lib/routes/qidian/chapter.ts
View File

@ -3,6 +3,7 @@ import { load } from 'cheerio';
import type { Route } from '@/types';
import { ViewType } from '@/types';
import got from '@/utils/got';
import { PRESETS } from '@/utils/header-generator';
import { parseDate } from '@/utils/parse-date';
export const route: Route = {
@ -32,18 +33,13 @@ export const route: Route = {
async function handler(ctx) {
const id = ctx.req.param('id');
const userAgent = 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Mobile/15E148 Safari/604.1';
const headers = {
'User-Agent': userAgent,
};
const response = await got(`https://m.qidian.com/book/${id}.html`, { headers });
const response = await got(`https://m.qidian.com/book/${id}.html`, { headerGeneratorOptions: PRESETS.MODERN_IOS });
const $ = load(response.data);
const name = $('meta[property="og:title"]').attr('content');
const coverUrl = `https:${$('.detail__header-cover__img').attr('src')}`;
const { data: catalog } = await got(`https://m.qidian.com/book/${id}/catalog/`, { headers });
const { data: catalog } = await got(`https://m.qidian.com/book/${id}/catalog/`, { headerGeneratorOptions: PRESETS.MODERN_IOS });
const $c = load(catalog);
const { pageContext } = JSON.parse($c('#vite-plugin-ssr_pageContext').text());

7
lib/routes/qidian/forum.ts
View File

@ -2,10 +2,7 @@ import { load } from 'cheerio';
import type { Route } from '@/types';
import got from '@/utils/got';
const headers = {
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Mobile/15E148 Safari/604.1',
};
import { PRESETS } from '@/utils/header-generator';
export const route: Route = {
path: '/forum/:id',
@ -35,7 +32,7 @@ async function handler(ctx) {
// Reason: forum.qidian.com redirects and PC site has anti-bot JS challenge;
// mobile book page embeds forum posts via seoBookCirclePost
const response = await got(`https://m.qidian.com/book/${id}.html`, { headers });
const response = await got(`https://m.qidian.com/book/${id}.html`, { headerGeneratorOptions: PRESETS.MODERN_IOS });
const $ = load(response.data);
const { pageContext } = JSON.parse($('#vite-plugin-ssr_pageContext').text());
const pageData = pageContext.pageProps.pageData;

7
lib/routes/qidian/free-next.ts
View File

@ -2,10 +2,7 @@ import { load } from 'cheerio';
import type { Route } from '@/types';
import got from '@/utils/got';
const headers = {
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Mobile/15E148 Safari/604.1',
};
import { PRESETS } from '@/utils/header-generator';
export const route: Route = {
path: '/free-next/:type?',
@ -40,7 +37,7 @@ async function handler(ctx) {
const title = isMM ? '起点女生网' : '起点中文网';
// Reason: PC site (www.qidian.com) returns anti-bot JS challenge; mobile site has SSR data
const response = await got('https://m.qidian.com/free', { headers });
const response = await got('https://m.qidian.com/free', { headerGeneratorOptions: PRESETS.MODERN_IOS });
const $ = load(response.data);
const { pageContext } = JSON.parse($('#vite-plugin-ssr_pageContext').text());
const pageData = pageContext.pageProps.pageData;

7
lib/routes/qidian/free.ts
View File

@ -2,10 +2,7 @@ import { load } from 'cheerio';
import type { Route } from '@/types';
import got from '@/utils/got';
const headers = {
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Mobile/15E148 Safari/604.1',
};
import { PRESETS } from '@/utils/header-generator';
export const route: Route = {
path: '/free/:type?',
@ -40,7 +37,7 @@ async function handler(ctx) {
const title = isMM ? '起点女生网' : '起点中文网';
// Reason: PC site (www.qidian.com) returns anti-bot JS challenge; mobile site has SSR data
const response = await got('https://m.qidian.com/free', { headers });
const response = await got('https://m.qidian.com/free', { headerGeneratorOptions: PRESETS.MODERN_IOS });
const $ = load(response.data);
const { pageContext } = JSON.parse($('#vite-plugin-ssr_pageContext').text());
const pageData = pageContext.pageProps.pageData;

6
lib/routes/sciencedirect/call-for-paper.tsx
View File

@ -26,11 +26,7 @@ export const route: Route = {
async function handler(ctx) {
const { subject = '' } = ctx.req.param();
const apiUrl = `https://www.sciencedirect.com/browse/calls-for-papers?subject=${subject}`;
const headers = {
accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', // need this to avoid 403, 503 error
};
const response = await got(apiUrl, { headers });
const response = await got(apiUrl);
const $ = load(response.body);
const scriptJSON = $('script[data-iso-key="_0"]').text();

6
lib/routes/the/index.ts
View File

@ -21,11 +21,7 @@ export const handler = async (ctx): Promise<Data> => {
const slug = parseSlug(filter);
const targetUrl = slug ? `${baseUrl}/${slug}` : `${baseUrl}/${defaultFeedPath}`;
const { data: response } = await got(targetUrl, {
headers: {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
},
});
const { data: response } = await got(targetUrl);
const $ = load(response);

3
lib/routes/vimeo/channel.ts
View File

@ -3,6 +3,7 @@ import { load } from 'cheerio';
import type { Route } from '@/types';
import cache from '@/utils/cache';
import got from '@/utils/got';
import { PRESETS } from '@/utils/header-generator';
import { parseDate } from '@/utils/parse-date';
import { renderDescription } from './templates/description';
@ -64,8 +65,8 @@ async function handler(ctx) {
url: `https://vimeo.com${link}/description?breeze=1`,
headers: {
'X-Requested-With': 'XMLHttpRequest',
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) ',
},
headerGeneratorOptions: PRESETS.MODERN_IOS,
});
const articledata = response2.data;
const $2 = load(articledata);

6
lib/routes/youmemark/index.ts
View File

@ -36,11 +36,7 @@ async function handler(ctx): Promise<Data> {
const userid = ctx.req.param('userid');
const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 10;
const response = await ofetch(`https://youmemark.com/user/${userid}`, {
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
},
});
const response = await ofetch(`https://youmemark.com/user/${userid}`);
const $ = load(response);