fix(route/xueqiu): fix stock_info failure caused by WAF challenge (#22346)
The /xueqiu/stock_info route was broken by Alibaba Cloud WAF: the cookie acquisition could not pass the challenge, and the data request was sent to xueqiu.com which the WAF blocks for these endpoints. - cookies.ts: allow the `script` resource type in the page.route filter so the WAF challenge scripts can execute, enabling cookie extraction via Patchright - stock-info.ts: request data from api.xueqiu.com instead of xueqiu.com to bypass the WAF - stock-info.ts: align the supported types with the tabs on the stock page (all / discuss / trans / news / announcement). all / discuss / trans use the symbol search endpoint; news / announcement use the stock timeline endpoint. The previously broken research type is removed - stock-info.ts: fetch the stock name from the lightweight quote API since it is rendered client-side and cannot be scraped from the HTML - stock-info.ts: throw InvalidParameterError for unsupported types Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
sirius60111
GitHub
parent
aa46e600e1
commit
a510ea3f24
|
|
@ -11,7 +11,8 @@ export const parseToken = (link: string) =>
|
|||
const page = await context.newPage();
|
||||
await page.route('**/*', (route) => {
|
||||
const request = route.request();
|
||||
request.resourceType() === 'document' ? route.continue() : route.abort();
|
||||
const type = request.resourceType();
|
||||
(type === 'document' || type === 'script') ? route.continue() : route.abort();
|
||||
});
|
||||
await page.goto(link, {
|
||||
waitUntil: 'domcontentloaded',
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { load } from 'cheerio';
|
||||
import queryString from 'query-string';
|
||||
import sanitizeHtml from 'sanitize-html';
|
||||
|
||||
import InvalidParameterError from '@/errors/types/invalid-parameter';
|
||||
import { parseToken } from '@/routes/xueqiu/cookies';
|
||||
import type { Route } from '@/types';
|
||||
import got from '@/utils/got';
|
||||
|
|
@ -15,7 +15,7 @@ export const route: Route = {
|
|||
features: {
|
||||
requireConfig: false,
|
||||
requirePuppeteer: false,
|
||||
antiCrawler: false,
|
||||
antiCrawler: true,
|
||||
supportBT: false,
|
||||
supportPodcast: false,
|
||||
supportScihub: false,
|
||||
|
|
@ -29,36 +29,49 @@ export const route: Route = {
|
|||
name: '股票信息',
|
||||
maintainers: ['YuYang'],
|
||||
handler,
|
||||
description: `| 公告 | 新闻 | 研报 |
|
||||
| ------------ | ---- | -------- |
|
||||
| announcement | news | research |`,
|
||||
description: `| 全部 | 讨论 | 交易 | 资讯 | 公告 |
|
||||
| ---- | ------- | ----- | ---- | ------------ |
|
||||
| all | discuss | trans | news | announcement |`,
|
||||
};
|
||||
|
||||
// The two endpoints below correspond to the tabs on the stock page (xueqiu.com/S/:id).
|
||||
// `source` is the API query value; `label` is the human-readable name shown in the feed title.
|
||||
// `all` / `discuss` / `trans` are served by the search endpoint; `news` / `announcement`
|
||||
// are served by the timeline endpoint (the search endpoint ignores these two sources).
|
||||
const typeMap = {
|
||||
all: { source: 'all', label: '全部', endpoint: 'search' },
|
||||
discuss: { source: 'user', label: '讨论', endpoint: 'search' },
|
||||
trans: { source: 'trans', label: '交易', endpoint: 'search' },
|
||||
news: { source: '自选股新闻', label: '资讯', endpoint: 'timeline' },
|
||||
announcement: { source: '公告', label: '公告', endpoint: 'timeline' },
|
||||
};
|
||||
|
||||
async function handler(ctx) {
|
||||
const id = ctx.req.param('id');
|
||||
const type = ctx.req.param('type') || 'announcement';
|
||||
const count = 10;
|
||||
const page = 1;
|
||||
const typename = {
|
||||
announcement: '公告',
|
||||
news: '自选股新闻',
|
||||
research: '研报',
|
||||
all: 'all',
|
||||
};
|
||||
const source = typename[type];
|
||||
if (!Object.hasOwn(typeMap, type)) {
|
||||
throw new InvalidParameterError(`Invalid type: ${type}. Supported types: ${Object.keys(typeMap).join(', ')}`);
|
||||
}
|
||||
const { source, label, endpoint } = typeMap[type];
|
||||
|
||||
const link = `https://xueqiu.com/S/${id}`;
|
||||
const res1 = await got({
|
||||
const cookie = await parseToken(link);
|
||||
|
||||
// Fetch the stock name from the lightweight quote API (the name is rendered
|
||||
// client-side on the page, so it cannot be scraped from the static HTML)
|
||||
const quoteRes = await got({
|
||||
method: 'get',
|
||||
url: link,
|
||||
url: 'https://stock.xueqiu.com/v5/stock/quote.json',
|
||||
searchParams: queryString.stringify({ symbol: id }),
|
||||
headers: {
|
||||
Cookie: cookie,
|
||||
Referer: link,
|
||||
},
|
||||
});
|
||||
const stock_name = quoteRes.data.data?.quote?.name || id;
|
||||
|
||||
const token = await parseToken(link);
|
||||
const $ = load(res1.data); // 使用 cheerio 加载返回的 HTML
|
||||
const stock_name = $('.stock-name').text().split('(', 1)[0];
|
||||
|
||||
let query_url = 'https://xueqiu.com/statuses';
|
||||
query_url += source === 'all' ? '/search.json' : '/stock_timeline.json';
|
||||
// Use the api.xueqiu.com domain; the WAF blocks these endpoints on xueqiu.com
|
||||
const query_url = endpoint === 'search' ? 'https://api.xueqiu.com/query/v1/symbol/search/status.json' : 'https://api.xueqiu.com/statuses/stock_timeline.json';
|
||||
|
||||
const res2 = await got({
|
||||
method: 'get',
|
||||
|
|
@ -67,23 +80,23 @@ async function handler(ctx) {
|
|||
symbol_id: id,
|
||||
symbol: id,
|
||||
source,
|
||||
count,
|
||||
page,
|
||||
sort: 'alpha',
|
||||
count: 10,
|
||||
page: 1,
|
||||
sort: endpoint === 'search' ? 'time' : 'alpha',
|
||||
comment: '0',
|
||||
hl: '0',
|
||||
}),
|
||||
headers: {
|
||||
Cookie: token,
|
||||
Cookie: cookie,
|
||||
Referer: link,
|
||||
},
|
||||
});
|
||||
|
||||
const data = res2.data.list;
|
||||
return {
|
||||
title: `${id} ${stock_name} - ${source}`,
|
||||
title: `${id} ${stock_name} - ${label}`,
|
||||
link,
|
||||
description: `${stock_name} - ${source}`,
|
||||
description: `${stock_name} - ${label}`,
|
||||
item: data.map((item) => ({
|
||||
title: item.title || sanitizeHtml(item.description, { allowedTags: [], allowedAttributes: {} }),
|
||||
description: item.description,
|
||||
|
|
|
|||
Loading…
Reference in New Issue