diff --git a/lib/routes/36kr/hot-list.ts b/lib/routes/36kr/hot-list.ts index 8aeb7e055..c28b4f287 100644 --- a/lib/routes/36kr/hot-list.ts +++ b/lib/routes/36kr/hot-list.ts @@ -4,7 +4,7 @@ import cache from '@/utils/cache'; import got from '@/utils/got'; import { parseDate } from '@/utils/parse-date'; -import { ProcessItem, rootUrl } from './utils'; +import { getWafTokenId, ProcessItem, rootUrl } from './utils'; const categories = { 24: { @@ -70,9 +70,13 @@ async function handler(ctx) { const currentUrl = category === '24' ? rootUrl : `${rootUrl}/hot-list/catalog`; + const wafTokenId = await getWafTokenId(); const response = await got({ method: 'get', url: currentUrl, + headers: { + Cookie: `_waftokenid=${wafTokenId}`, + }, }); const data = getProperty(JSON.parse(response.data.match(/window.initialState=({.*})/)[1]), categories[category].key); diff --git a/lib/routes/36kr/utils.ts b/lib/routes/36kr/utils.ts index 9c5e8d101..5149e1ae2 100644 --- a/lib/routes/36kr/utils.ts +++ b/lib/routes/36kr/utils.ts @@ -1,21 +1,20 @@ import { load } from 'cheerio'; import CryptoJS from 'crypto-js'; -import got from '@/utils/got'; +import { solveWafChallenge } from '@/routes/juejin/utils'; +import cache from '@/utils/cache'; +import ofetch from '@/utils/ofetch'; -const rootUrl = 'https://www.36kr.com'; +export const rootUrl = 'https://www.36kr.com'; -const ProcessItem = (item, tryGet) => +export const ProcessItem = (item, tryGet) => tryGet(item.link, async () => { - const detailResponse = await got({ - method: 'get', - url: item.link, - }); + const detailResponse = await ofetch(item.link); - const cipherTextList = detailResponse.data.match(/{"state":"(.*)","isEncrypt":true}/) ?? []; + const cipherTextList = detailResponse.match(/{"state":"(.*)","isEncrypt":true}/) ?? []; if (cipherTextList.length === 0) { - const $ = load(detailResponse.body); + const $ = load(detailResponse); item.description = $('div.articleDetailContent').html(); } else { const key = CryptoJS.enc.Utf8.parse('efabccee-b754-4c'); @@ -33,4 +32,33 @@ const ProcessItem = (item, tryGet) => return item; }); -export { ProcessItem, rootUrl }; +export const getWafTokenId = () => + cache.tryGet( + '36kr:_waftokenid', + async () => { + const captchaResponse = await ofetch(rootUrl); + + const $ = load(captchaResponse); + const payload = $('script') + .text() + .match(/atob\('(.*?)'\)\),/)?.[1]; + const response = solveWafChallenge(payload); + + const tokenIdResponse = await ofetch.raw(rootUrl, { + headers: { + Cookie: `_wafchallengeid=${response};`, + }, + redirect: 'manual', + }); + + const _wafTokenId = tokenIdResponse.headers + .getSetCookie() + .find((cookie) => cookie.startsWith('_waftokenid=')) + ?.split(';')[0] + .split('=')[1]; + + return _wafTokenId as string; + }, + 300, // server-provided value + false + ); diff --git a/lib/routes/juejin/utils.ts b/lib/routes/juejin/utils.ts index 6f87b7f94..0fd335158 100644 --- a/lib/routes/juejin/utils.ts +++ b/lib/routes/juejin/utils.ts @@ -17,7 +17,12 @@ const s256 = (s1: Uint8Array, s2: string) => { return sha.digest('hex'); }; -const solveWafChallenge = (cs) => { +/** + * Solve _wafchallengeid + * @param cs - base64 encoded challenge string {"v":{"a":"...", "b":"timestamp", "c":"..."}, "s":"..."} + * @returns base64 encoded solved challenge string {"v":{"a":"...", "b":"timestamp", "c":"..."}, "s":"...", "d":"solution"} + */ +export const solveWafChallenge = (cs: string) => { const c = JSON.parse(Buffer.from(cs, 'base64').toString()); const prefix = b64tou8a(c.v.a); const expect = b64tohex(c.v.c);