diff --git a/lib/routes/changba/user.tsx b/lib/routes/changba/user.tsx index c601b2d9e..26e6210eb 100644 --- a/lib/routes/changba/user.tsx +++ b/lib/routes/changba/user.tsx @@ -5,8 +5,7 @@ import type { Route } from '@/types'; import { ViewType } from '@/types'; import cache from '@/utils/cache'; import got from '@/utils/got'; - -const headers = { 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1' }; +import { PRESETS } from '@/utils/header-generator'; export const route: Route = { path: '/:userid', @@ -38,7 +37,7 @@ async function handler(ctx) { const response = await got({ method: 'get', url, - headers, + headerGeneratorOptions: PRESETS.MODERN_IOS, }); const data = response.data; const $ = load(data); @@ -54,7 +53,7 @@ async function handler(ctx) { const result = await got({ method: 'get', url: link, - headers, + headerGeneratorOptions: PRESETS.MODERN_IOS, }); const re = /workid: '\d+'/; diff --git a/lib/routes/ddosi/category.ts b/lib/routes/ddosi/category.ts index 5c8ee1a93..9e1015e0e 100644 --- a/lib/routes/ddosi/category.ts +++ b/lib/routes/ddosi/category.ts @@ -1,8 +1,8 @@ import { load } from 'cheerio'; -import { config } from '@/config'; import type { Route } from '@/types'; import got from '@/utils/got'; +import { PRESETS } from '@/utils/header-generator'; import { parseDate } from '@/utils/parse-date'; export const route: Route = { @@ -25,22 +25,21 @@ export const route: Route = { }, ], name: '分类', - maintainers: [], + maintainers: ['XinRoom'], handler, url: 'ddosi.org/', }; async function handler(ctx) { const url = 'https://www.ddosi.org/category'; - const userAgent = config.ua || 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1'; const category = ctx.req.param('category'); const response = await got({ method: 'get', url: `${url}/${category}/`, headers: { - 'User-Agent': userAgent, Referer: url, }, + headerGeneratorOptions: PRESETS.MODERN_IOS, }); const $ = load(response.data); const list = $('main>article').toArray(); diff --git a/lib/routes/ddosi/index.ts b/lib/routes/ddosi/index.ts index c93a8db9f..5e7ec5b9f 100644 --- a/lib/routes/ddosi/index.ts +++ b/lib/routes/ddosi/index.ts @@ -1,19 +1,20 @@ import { load } from 'cheerio'; -import { config } from '@/config'; import type { Route } from '@/types'; import got from '@/utils/got'; +import { PRESETS } from '@/utils/header-generator'; import { parseDate } from '@/utils/parse-date'; export const route: Route = { path: '/', + example: '/ddosi', radar: [ { source: ['ddosi.org/'], target: '', }, ], - name: 'Unknown', + name: '首页', maintainers: ['XinRoom'], handler, url: 'ddosi.org/', @@ -21,14 +22,13 @@ export const route: Route = { async function handler() { const url = 'https://www.ddosi.org/'; - const userAgent = config.ua || 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1'; const response = await got({ method: 'get', url: String(url), headers: { - 'User-Agent': userAgent, Referer: url, }, + headerGeneratorOptions: PRESETS.MODERN_IOS, }); const $ = load(response.data); const list = $('main>article').toArray(); diff --git a/lib/routes/dianping/user.ts b/lib/routes/dianping/user.ts index 3503b6602..6698f1aae 100644 --- a/lib/routes/dianping/user.ts +++ b/lib/routes/dianping/user.ts @@ -1,5 +1,6 @@ import { config } from '@/config'; import type { Route } from '@/types'; +import { PRESETS } from '@/utils/header-generator'; import ofetch from '@/utils/ofetch'; export const route: Route = { @@ -58,12 +59,10 @@ const starMap: Record = { async function handler(ctx) { const id = ctx.req.param('id'); - const userAgent = 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1'; const userPage = `https://m.dianping.com/userprofile/${id}`; const cookie = config.dianping.cookie; const headers: Record = { - 'User-Agent': userAgent, Referer: userPage, }; @@ -73,6 +72,7 @@ async function handler(ctx) { const pageResponse = await ofetch(userPage, { headers, + headerGeneratorOptions: PRESETS.MODERN_IOS, }); const nickNameReg = /window\.nickName = "(.*?)"/g; @@ -80,6 +80,7 @@ async function handler(ctx) { const response = await ofetch(`https://m.dianping.com/member/ajax/NobleUserFeeds?userId=${id}`, { headers, + headerGeneratorOptions: PRESETS.MODERN_IOS, }); const data = response.data; diff --git a/lib/routes/hyperdash/utils.ts b/lib/routes/hyperdash/utils.ts index f36f29798..b3e9c0fec 100644 --- a/lib/routes/hyperdash/utils.ts +++ b/lib/routes/hyperdash/utils.ts @@ -21,7 +21,6 @@ export async function fetchTopTraders(): Promise { const traders: TraderData[] = await ofetch(apiUrl, { headers: { 'x-api-key': 'hyperdash_public_7vN3mK8pQ4wX2cL9hF5tR1bY6gS0jD', - 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36', }, }); diff --git a/lib/routes/jandan/utils.ts b/lib/routes/jandan/utils.ts index 1b37789c0..c2b702c5b 100644 --- a/lib/routes/jandan/utils.ts +++ b/lib/routes/jandan/utils.ts @@ -4,15 +4,12 @@ import type { DataItem } from '@/types'; import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; -export const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36'; - /** * Extract page ID from script tags in HTML */ export const extractPageId = async (url: string, referer: string): Promise => { const response = await ofetch(url, { headers: { - 'User-Agent': USER_AGENT, Referer: referer, Accept: 'application/json, text/plain, */*', }, @@ -39,7 +36,6 @@ export const handleTopSection = async (rootUrl: string, type: string): Promise<{ const apiUrl = `${rootUrl}/api/top/${type}`; const response = await ofetch(apiUrl, { headers: { - 'User-Agent': USER_AGENT, Referer: rootUrl, Accept: 'application/json, text/plain, */*', }, @@ -114,7 +110,6 @@ export const handleForumSection = async (rootUrl: string): Promise<{ title: stri const apiUrl = `${rootUrl}/api/forum/posts/${forumId}?page=1`; const forumData = await ofetch(apiUrl, { headers: { - 'User-Agent': USER_AGENT, Referer: currentUrl, Accept: 'application/json, text/plain, */*', }, @@ -174,7 +169,6 @@ export const handleCommentSection = async (rootUrl: string, category: string): P const response = await ofetch(currentUrl, { headers: { - 'User-Agent': USER_AGENT, Referer: rootUrl, Accept: 'application/json, text/plain, */*', }, @@ -200,7 +194,6 @@ export const handleCommentSection = async (rootUrl: string, category: string): P const apiUrl = `${rootUrl}/api/comment/post/${pageId}?order=desc&page=1`; const commentsData = await ofetch(apiUrl, { headers: { - 'User-Agent': USER_AGENT, Referer: currentUrl, Accept: 'application/json, text/plain, */*', }, diff --git a/lib/routes/kuaidi100/utils.ts b/lib/routes/kuaidi100/utils.ts index 7d6bd38ac..9fa313279 100644 --- a/lib/routes/kuaidi100/utils.ts +++ b/lib/routes/kuaidi100/utils.ts @@ -230,7 +230,6 @@ export default { headers: { Referer: 'https://www.kuaidi100.com/', 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,ja;q=0.7', - 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36', Cookie: `${cookie.globacsrftoken}; ${cookie.csrf}; ${cookie.wwwid}; ${cookie.dasddocHref}; ${cookie.dasddocReferrer}; ${ cookie.dasddocTitl }; addcom=${number}; addnu=${id}; snt_query_meta=${queryMeta}; sortStatus=0; Hm_lpvt_22ea01af58ba2be0fec7c11b25e88e6c=${timestamp}; Hm_lvt_22ea01af58ba2be0fec7c11b25e88e6c=${timestamp - 1642}`, diff --git a/lib/routes/qiche365/recall.ts b/lib/routes/qiche365/recall.ts index f1f557650..c1a49d266 100644 --- a/lib/routes/qiche365/recall.ts +++ b/lib/routes/qiche365/recall.ts @@ -7,8 +7,6 @@ import timezone from '@/utils/timezone'; const baseUrl = 'https://www.qiche365.org.cn'; -const userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'; - export const route: Route = { path: '/recall/:channel', name: '汽车召回', @@ -34,7 +32,6 @@ async function handler(ctx): Promise { // second request with those cookies returns the actual JSON data. const initResponse = await ofetch.raw(targetUrl, { headers: { - 'User-Agent': userAgent, 'Accept-Language': 'zh-CN,zh;q=0.9', }, ignoreResponseError: true, @@ -44,7 +41,6 @@ async function handler(ctx): Promise { const { html } = await ofetch(targetUrl, { headers: { - 'User-Agent': userAgent, 'Accept-Language': 'zh-CN,zh;q=0.9', Cookie: cookies, }, diff --git a/lib/routes/qidian/author.tsx b/lib/routes/qidian/author.tsx index aa4b0e3cd..8dee269a1 100644 --- a/lib/routes/qidian/author.tsx +++ b/lib/routes/qidian/author.tsx @@ -2,10 +2,7 @@ import { load } from 'cheerio'; import type { Route } from '@/types'; import got from '@/utils/got'; - -const headers = { - 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Mobile/15E148 Safari/604.1', -}; +import { PRESETS } from '@/utils/header-generator'; export const route: Route = { path: '/author/:id', @@ -35,7 +32,7 @@ async function handler(ctx) { const currentUrl = `https://my.qidian.com/author/${id}/`; // Reason: PC site (my.qidian.com) returns anti-bot JS challenge; mobile site has SSR data - const response = await got(`https://m.qidian.com/author/${id}/`, { headers }); + const response = await got(`https://m.qidian.com/author/${id}/`, { headerGeneratorOptions: PRESETS.MODERN_IOS }); const $ = load(response.data); const { pageContext } = JSON.parse($('#vite-plugin-ssr_pageContext').text()); const pageData = pageContext.pageProps.pageData; diff --git a/lib/routes/qidian/chapter.ts b/lib/routes/qidian/chapter.ts index 87f33b11e..9389c799c 100644 --- a/lib/routes/qidian/chapter.ts +++ b/lib/routes/qidian/chapter.ts @@ -3,6 +3,7 @@ import { load } from 'cheerio'; import type { Route } from '@/types'; import { ViewType } from '@/types'; import got from '@/utils/got'; +import { PRESETS } from '@/utils/header-generator'; import { parseDate } from '@/utils/parse-date'; export const route: Route = { @@ -32,18 +33,13 @@ export const route: Route = { async function handler(ctx) { const id = ctx.req.param('id'); - const userAgent = 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Mobile/15E148 Safari/604.1'; - const headers = { - 'User-Agent': userAgent, - }; - - const response = await got(`https://m.qidian.com/book/${id}.html`, { headers }); + const response = await got(`https://m.qidian.com/book/${id}.html`, { headerGeneratorOptions: PRESETS.MODERN_IOS }); const $ = load(response.data); const name = $('meta[property="og:title"]').attr('content'); const coverUrl = `https:${$('.detail__header-cover__img').attr('src')}`; - const { data: catalog } = await got(`https://m.qidian.com/book/${id}/catalog/`, { headers }); + const { data: catalog } = await got(`https://m.qidian.com/book/${id}/catalog/`, { headerGeneratorOptions: PRESETS.MODERN_IOS }); const $c = load(catalog); const { pageContext } = JSON.parse($c('#vite-plugin-ssr_pageContext').text()); diff --git a/lib/routes/qidian/forum.ts b/lib/routes/qidian/forum.ts index 03e1932d5..3c7bdb686 100644 --- a/lib/routes/qidian/forum.ts +++ b/lib/routes/qidian/forum.ts @@ -2,10 +2,7 @@ import { load } from 'cheerio'; import type { Route } from '@/types'; import got from '@/utils/got'; - -const headers = { - 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Mobile/15E148 Safari/604.1', -}; +import { PRESETS } from '@/utils/header-generator'; export const route: Route = { path: '/forum/:id', @@ -35,7 +32,7 @@ async function handler(ctx) { // Reason: forum.qidian.com redirects and PC site has anti-bot JS challenge; // mobile book page embeds forum posts via seoBookCirclePost - const response = await got(`https://m.qidian.com/book/${id}.html`, { headers }); + const response = await got(`https://m.qidian.com/book/${id}.html`, { headerGeneratorOptions: PRESETS.MODERN_IOS }); const $ = load(response.data); const { pageContext } = JSON.parse($('#vite-plugin-ssr_pageContext').text()); const pageData = pageContext.pageProps.pageData; diff --git a/lib/routes/qidian/free-next.ts b/lib/routes/qidian/free-next.ts index 412caacf4..05722cd15 100644 --- a/lib/routes/qidian/free-next.ts +++ b/lib/routes/qidian/free-next.ts @@ -2,10 +2,7 @@ import { load } from 'cheerio'; import type { Route } from '@/types'; import got from '@/utils/got'; - -const headers = { - 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Mobile/15E148 Safari/604.1', -}; +import { PRESETS } from '@/utils/header-generator'; export const route: Route = { path: '/free-next/:type?', @@ -40,7 +37,7 @@ async function handler(ctx) { const title = isMM ? '起点女生网' : '起点中文网'; // Reason: PC site (www.qidian.com) returns anti-bot JS challenge; mobile site has SSR data - const response = await got('https://m.qidian.com/free', { headers }); + const response = await got('https://m.qidian.com/free', { headerGeneratorOptions: PRESETS.MODERN_IOS }); const $ = load(response.data); const { pageContext } = JSON.parse($('#vite-plugin-ssr_pageContext').text()); const pageData = pageContext.pageProps.pageData; diff --git a/lib/routes/qidian/free.ts b/lib/routes/qidian/free.ts index a0d46f3c2..10a19f0f6 100644 --- a/lib/routes/qidian/free.ts +++ b/lib/routes/qidian/free.ts @@ -2,10 +2,7 @@ import { load } from 'cheerio'; import type { Route } from '@/types'; import got from '@/utils/got'; - -const headers = { - 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Mobile/15E148 Safari/604.1', -}; +import { PRESETS } from '@/utils/header-generator'; export const route: Route = { path: '/free/:type?', @@ -40,7 +37,7 @@ async function handler(ctx) { const title = isMM ? '起点女生网' : '起点中文网'; // Reason: PC site (www.qidian.com) returns anti-bot JS challenge; mobile site has SSR data - const response = await got('https://m.qidian.com/free', { headers }); + const response = await got('https://m.qidian.com/free', { headerGeneratorOptions: PRESETS.MODERN_IOS }); const $ = load(response.data); const { pageContext } = JSON.parse($('#vite-plugin-ssr_pageContext').text()); const pageData = pageContext.pageProps.pageData; diff --git a/lib/routes/sciencedirect/call-for-paper.tsx b/lib/routes/sciencedirect/call-for-paper.tsx index f1903b0b4..4729c5132 100644 --- a/lib/routes/sciencedirect/call-for-paper.tsx +++ b/lib/routes/sciencedirect/call-for-paper.tsx @@ -26,11 +26,7 @@ export const route: Route = { async function handler(ctx) { const { subject = '' } = ctx.req.param(); const apiUrl = `https://www.sciencedirect.com/browse/calls-for-papers?subject=${subject}`; - const headers = { - accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', - 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', // need this to avoid 403, 503 error - }; - const response = await got(apiUrl, { headers }); + const response = await got(apiUrl); const $ = load(response.body); const scriptJSON = $('script[data-iso-key="_0"]').text(); diff --git a/lib/routes/the/index.ts b/lib/routes/the/index.ts index cdc93f993..94f7c9295 100644 --- a/lib/routes/the/index.ts +++ b/lib/routes/the/index.ts @@ -21,11 +21,7 @@ export const handler = async (ctx): Promise => { const slug = parseSlug(filter); const targetUrl = slug ? `${baseUrl}/${slug}` : `${baseUrl}/${defaultFeedPath}`; - const { data: response } = await got(targetUrl, { - headers: { - 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - }, - }); + const { data: response } = await got(targetUrl); const $ = load(response); diff --git a/lib/routes/vimeo/channel.ts b/lib/routes/vimeo/channel.ts index 8d6d169f5..eb34fd65b 100644 --- a/lib/routes/vimeo/channel.ts +++ b/lib/routes/vimeo/channel.ts @@ -3,6 +3,7 @@ import { load } from 'cheerio'; import type { Route } from '@/types'; import cache from '@/utils/cache'; import got from '@/utils/got'; +import { PRESETS } from '@/utils/header-generator'; import { parseDate } from '@/utils/parse-date'; import { renderDescription } from './templates/description'; @@ -64,8 +65,8 @@ async function handler(ctx) { url: `https://vimeo.com${link}/description?breeze=1`, headers: { 'X-Requested-With': 'XMLHttpRequest', - 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) ', }, + headerGeneratorOptions: PRESETS.MODERN_IOS, }); const articledata = response2.data; const $2 = load(articledata); diff --git a/lib/routes/youmemark/index.ts b/lib/routes/youmemark/index.ts index 90236327d..1d272abbd 100644 --- a/lib/routes/youmemark/index.ts +++ b/lib/routes/youmemark/index.ts @@ -36,11 +36,7 @@ async function handler(ctx): Promise { const userid = ctx.req.param('userid'); const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 10; - const response = await ofetch(`https://youmemark.com/user/${userid}`, { - headers: { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - }, - }); + const response = await ofetch(`https://youmemark.com/user/${userid}`); const $ = load(response);