From f3cf86cd72e96a9ca84fd10967186e89e59c8bfc Mon Sep 17 00:00:00 2001 From: Rongrong Date: Mon, 22 Apr 2024 02:19:33 +0800 Subject: [PATCH] fix(core/utils/wechat-mp): empty description when request blocked by WAF (#15294) * fix(core/utils/wechat-mp): empty description when request blocked by WAF Silently falling back to the fallback method would generate empty description. Fixed by throwing an error when failed to extract item_show_type from the page. Some misc warnings are also added to make bug reporting easier. Signed-off-by: Rongrong * fix(core/utils/wechat-mp): err msg for unknown page Signed-off-by: Rongrong * test(core/utils/wechat-mp): fix coverage Signed-off-by: Rongrong * fix(core/utils/wechat-mp): ESLint no-await-in-loop Signed-off-by: Rongrong * fix(core/utils/wechat-mp): normalizeUrl: & -> & Signed-off-by: Rongrong * feat(core/utils/wechat-mp): do not prompt raising an issue when blocked by WAF Signed-off-by: Rongrong * feat(core/utils/wechat-mp): recognize deleted page Signed-off-by: Rongrong * feat(core/utils/wechat-mp): log when error Prevent errors from being completely suppressed if the caller catches them. Signed-off-by: Rongrong --------- Signed-off-by: Rongrong --- lib/setup.test.ts | 67 +++++++++++++++++++++++++- lib/utils/wechat-mp.test.ts | 79 ++++++++++++++++++++++++++++-- lib/utils/wechat-mp.ts | 96 ++++++++++++++++++++++++++++++------- 3 files changed, 218 insertions(+), 24 deletions(-) diff --git a/lib/setup.test.ts b/lib/setup.test.ts index a06f67c51..ab7436a3c 100644 --- a/lib/setup.test.ts +++ b/lib/setup.test.ts @@ -154,8 +154,71 @@ var ct = "${1_636_626_300}"; ) ) ), - http.get(`https://mp.weixin.qq.com/s/rsshub_test`, () => HttpResponse.text(genWeChatMpPage('', ''))), - http.get(`https://mp.weixin.qq.com/s?__biz=rsshub_test&mid=1&idx=1&sn=1`, () => HttpResponse.text(genWeChatMpPage('', ''))), + http.get(`https://mp.weixin.qq.com/s/rsshub_test`, () => HttpResponse.redirect(`https://mp.weixin.qq.com/rsshub_test/fallback`)), + http.get(`https://mp.weixin.qq.com/s?__biz=rsshub_test&mid=1&idx=1&sn=1`, () => HttpResponse.redirect(`https://mp.weixin.qq.com/rsshub_test/fallback`)), + http.get(`https://mp.weixin.qq.com/mp/rsshub_test/waf`, () => + HttpResponse.text( + ` + +Title + + + + + +
+ +
+ +
+
+

环境异常

+

当前环境异常，完成验证后即可继续访问。

+
+
+

+ 去验证 +

+
+
+` + ) + ), + http.get(`https://mp.weixin.qq.com/s/rsshub_test_hit_waf`, () => HttpResponse.redirect(`https://mp.weixin.qq.com/mp/rsshub_test/waf`)), + http.get(`https://mp.weixin.qq.com/s/unknown_page`, () => + HttpResponse.text( + ` + +Title + + + + + +

+Unknown paragraph +

+` + ) + ), + http.get(`https://mp.weixin.qq.com/s/deleted_page`, () => + HttpResponse.text( + ` + +Title + + + + + +

+该内容已被发布者删除 +

+` + ) + ), + http.get(`https://mp.weixin.qq.com/s/rsshub_test_redirect_no_location`, () => HttpResponse.text('', { status: 302 })), + http.get(`https://mp.weixin.qq.com/s/rsshub_test_recursive_redirect`, () => HttpResponse.redirect(`https://mp.weixin.qq.com/s/rsshub_test_recursive_redirect`)), http.get(`http://rsshub.test/headers`, ({ request }) => HttpResponse.json({ ...Object.fromEntries(request.headers.entries()), diff --git a/lib/utils/wechat-mp.test.ts b/lib/utils/wechat-mp.test.ts index 2343ee69c..2d24a2110 100644 --- a/lib/utils/wechat-mp.test.ts +++ b/lib/utils/wechat-mp.test.ts @@ -1,14 +1,16 @@ -import { describe, expect, it, vi } from 'vitest'; +import { describe, expect, it, vi, afterEach } from 'vitest'; import { load } from 'cheerio'; import Parser from 'rss-parser'; import InvalidParameterError from '@/errors/types/invalid-parameter'; -import { exportedForTestingOnly, fetchArticle, finishArticleItem, fixArticleContent, normalizeUrl } from '@/utils/wechat-mp'; -const { ExtractMetadata, showTypeMapReverse } = exportedForTestingOnly; +import { exportedForTestingOnly, WeChatMpError, fetchArticle, finishArticleItem, fixArticleContent, normalizeUrl } from '@/utils/wechat-mp'; +const { toggleWerror, ExtractMetadata, showTypeMapReverse } = exportedForTestingOnly; vi.mock('@/utils/request-rewriter', () => ({ default: null })); const { default: app } = await import('@/app'); const parser = new Parser(); +afterEach(() => toggleWerror(false)); + const expectedItem: { title: string; summary: string; @@ -324,8 +326,23 @@ describe('wechat-mp', () => { expect(normalizeUrl(somethingElseWithHash.replace('https://', 'http://'))).toBe(somethingElse); const notWechatMp = 'https://im.not.wechat.mp/and/an/error/is/expected'; - expect(() => normalizeUrl(notWechatMp)).toThrow(); + expect(() => normalizeUrl(notWechatMp)).toThrow('URL host must be "mp.weixin.qq.com"'); expect(normalizeUrl(notWechatMp, true)).toBe(notWechatMp); + + const unknownSearchParam = mpArticleRoot + '?unknown=param'; + toggleWerror(false); + expect(normalizeUrl(unknownSearchParam)).toBe(unknownSearchParam); + toggleWerror(true); + expect(() => normalizeUrl(unknownSearchParam)).toThrow('WarningAsError: unknown URL search parameters'); + + const unknownPath = mpRoot + '/unknown/path'; + toggleWerror(false); + expect(normalizeUrl(unknownPath)).toBe(unknownPath); + toggleWerror(true); + expect(() => normalizeUrl(unknownPath, true)).toThrow('WarningAsError: unknown URL path'); + + const ampEscapedUrl = longUrl.replaceAll('&', '&'); + expect(normalizeUrl(ampEscapedUrl)).toBe(longUrlShortened); }); it('fetchArticle_&_finishArticleItem_appMsg', async () => { @@ -405,6 +422,60 @@ describe('wechat-mp', () => { await testFetchArticleFinishArticleItem('/fallback', { setMpNameAsAuthor: true, skipLink: true }); }); + it('hit_waf', async () => { + try { + await fetchArticle('https://mp.weixin.qq.com/s/rsshub_test_hit_waf'); + expect.unreachable('Should throw an error'); + } catch (error) { + expect(error).toBeInstanceOf(WeChatMpError); + expect((error).message).not.toContain('console.log'); + expect((error).message).not.toContain('.style'); + expect((error).message).not.toContain('Consider raise an issue'); + expect((error).message).toContain('request blocked by WAF:'); + expect((error).message).toContain('/mp/rsshub_test/waf'); + expect((error).message).toContain('Title'); + expect((error).message).toContain('环境异常'); + } + }); + + it('unknown_page', async () => { + const unknownPageUrl = 'https://mp.weixin.qq.com/s/unknown_page'; + try { + await fetchArticle(unknownPageUrl); + expect.unreachable('Should throw an error'); + } catch (error) { + expect(error).toBeInstanceOf(WeChatMpError); + expect((error).message).not.toContain('console.log'); + expect((error).message).not.toContain('.style'); + expect((error).message).toContain('Consider raise an issue'); + expect((error).message).toContain('unknown page,'); + expect((error).message).toContain('Title Unknown paragraph'); + expect((error).message).toContain(unknownPageUrl); + } + }); + + it('deleted_page', async () => { + const deletedPageUrl = 'https://mp.weixin.qq.com/s/deleted_page'; + + try { + await fetchArticle(deletedPageUrl); + expect.unreachable('Should throw an error'); + } catch (error) { + expect(error).toBeInstanceOf(WeChatMpError); + expect((error).message).not.toContain('console.log'); + expect((error).message).not.toContain('.style'); + expect((error).message).not.toContain('Consider raise an issue'); + expect((error).message).toContain('deleted by author:'); + expect((error).message).toContain('Title 该内容已被发布者删除'); + expect((error).message).toContain(deletedPageUrl); + } + }); + + it('redirect', () => { + expect(fetchArticle('https://mp.weixin.qq.com/s/rsshub_test_redirect_no_location')).rejects.toThrow('redirect without location'); + expect(fetchArticle('https://mp.weixin.qq.com/s/rsshub_test_recursive_redirect')).rejects.toThrow('too many redirects'); + }); + it('route_test', async () => { try { await app.request('/test/wechat-mp'); diff --git a/lib/utils/wechat-mp.ts b/lib/utils/wechat-mp.ts index 268b41578..13fc6aa31 100644 --- a/lib/utils/wechat-mp.ts +++ b/lib/utils/wechat-mp.ts @@ -31,11 +31,36 @@ import { parseDate } from '@/utils/parse-date'; import cache from '@/utils/cache'; import logger from '@/utils/logger'; -const MAINTAINERS = ['Rongronggg9']; +class WeChatMpError extends Error { + constructor(message: string) { + super(message); + this.name = 'WeChatMpError'; + } +} -const warn = (reason: string, details: string) => - logger.warn(`wechat-mp: ${reason}: ${details}, -consider raise an issue (mentioning ${MAINTAINERS.join(', ')}) with the article URL for further investigation`); +const MAINTAINERS = ['@Rongronggg9']; + +const formatLogNoMention = (...params: string[]): string => `wechat-mp: ${params.join(': ')}`; +const formatLog = (...params: string[]): string => `${formatLogNoMention(...params)} +Consider raise an issue (mentioning ${MAINTAINERS.join(', ')}) with the article URL for further investigation`; +let warn = (...params: string[]) => logger.warn(formatLog(...params)); +const error = (...params: string[]): never => { + const msg = formatLog(...params); + logger.error(msg); + throw new WeChatMpError(msg); +}; +const errorNoMention = (...params: string[]): never => { + const msg = formatLogNoMention(...params); + logger.error(msg); + throw new WeChatMpError(msg); +}; +const toggleWerror = (() => { + const onFunc = (...params: string[]) => error('WarningAsError', ...params); + const offFunc = warn; + return (on: boolean) => { + warn = on ? onFunc : offFunc; + }; +})(); const replaceReturnNewline = (() => { const returnRegExp = /\r|\\(r|x0d)/g; @@ -147,7 +172,7 @@ class ExtractMetadata { private static commonMetadataToBeExtracted = { showType: this.genExtractFunc('item_show_type', { valuePattern: '\\d+' }), realShowType: this.genExtractFunc('real_item_show_type', { valuePattern: '\\d+' }), - createTime: this.genExtractFunc('ct', { valuePattern: '\\d+' }), + createTime: this.genExtractFunc('ct', { valuePattern: '\\d+', allowNotFound: true }), sourceUrl: this.genExtractFunc('msg_source_url', { valuePattern: `https?://[^'"]*`, allowNotFound: true }), }; @@ -376,18 +401,21 @@ const fixArticleContent = (html?: string | Cheerio, skipImg = false) => // abtest_cookie, wx_header // Known params (temporary link): // src, timestamp, ver, signature, new (unessential) -const normalizeUrl = (url, bypassHostCheck = false) => { +const normalizeUrl = (url: string, bypassHostCheck = false) => { const oriUrl = url; + // already seen some weird urls with `&` escaped as `&`, so fix it + // calling fixUrl should always be safe since having `&` or `\x26` in a URL is meaningless + url = fixUrl(url); const urlObj = new URL(url); if (!bypassHostCheck && urlObj.host !== 'mp.weixin.qq.com') { - throw new Error('wechat-mp: URL host must be "mp.weixin.qq.com", but got ' + oriUrl); + error('URL host must be "mp.weixin.qq.com"', url); } urlObj.protocol = 'https:'; urlObj.hash = ''; // remove hash - if (/^\/s\/.+/.test(urlObj.pathname)) { + if (urlObj.pathname.startsWith('/s/')) { // a short link, just remove all the params urlObj.search = ''; - } else if (/^\/s$/.test(urlObj.pathname)) { + } else if (urlObj.pathname === '/s') { const biz = urlObj.searchParams.get('__biz'); const mid = urlObj.searchParams.get('mid') || urlObj.searchParams.get('appmsgid'); const idx = urlObj.searchParams.get('idx') || urlObj.searchParams.get('itemidx'); @@ -405,11 +433,11 @@ const normalizeUrl = (url, bypassHostCheck = false) => { // a temporary link, remove all unessential params urlObj.search = `?src=${src}×tamp=${timestamp}&ver=${ver}&signature=${signature}`; } else { - // unknown link, just let it go + warn('unknown URL search parameters', oriUrl); } } } else { - // IDK what it is, just let it go + warn('unknown URL path', oriUrl); } return urlObj.href; }; @@ -479,9 +507,11 @@ class PageParsers { } return page; }; - static dispatch = async ($: CheerioAPI) => { + static dispatch = async (html: string, url: string) => { + const $ = load(html); const commonMetadata = ExtractMetadata.common($); let page: Record; + let pageText: string, pageTextShort: string; switch (commonMetadata.showType) { case 'APP_MSG_PAGE': page = await PageParsers.appMsg($, commonMetadata); @@ -495,8 +525,24 @@ class PageParsers { case 'VIDEO_SHARE_PAGE': page = PageParsers.fallback($, commonMetadata); break; + case undefined: + $('script, style').remove(); + pageText = $('title, body').text().replaceAll(/\s+/g, ' ').trim(); + pageTextShort = pageText.slice(0, 25); + if (pageText.length >= 25 + '...'.length) { + pageTextShort = pageText.slice(0, 25); + pageTextShort += '...'; + } + if (pageText.includes('已被发布者删除')) { + errorNoMention('deleted by author', pageTextShort, url); + } else if (new URL(url).pathname.includes('captcha') || pageText.includes('环境异常')) { + errorNoMention('request blocked by WAF', pageTextShort, url); + } else { + error('unknown page, probably due to WAF', pageTextShort, url); + } + return {}; // just to make TypeScript happy, actually UNREACHABLE default: - warn('new showType, trying fallback method', `showType=${commonMetadata.showType}`); + warn('new showType, trying fallback method', `showType=${commonMetadata.showType}`, url); page = PageParsers.fallback($, commonMetadata); } const locationMetadata = ExtractMetadata.location($); @@ -517,6 +563,20 @@ class PageParsers { }; } +const redirectHelper = async (url: string, maxRedirects: number = 5) => { + maxRedirects--; + const raw = await ofetch.raw(url); + if ([301, 302, 303, 307, 308].includes(raw.status)) { + if (!raw.headers.has('location')) { + error('redirect without location', url); + } else if (maxRedirects <= 0) { + error('too many redirects', url); + } + return await redirectHelper(raw.headers.get('location'), maxRedirects); + } + return raw; +}; + /** * Fetch article and its metadata from WeChat MP (mp.weixin.qq.com). * @@ -528,9 +588,9 @@ class PageParsers { const fetchArticle = (url: string, bypassHostCheck: boolean = false) => { url = normalizeUrl(url, bypassHostCheck); return cache.tryGet(url, async () => { - const data = await ofetch(url); - const $ = load(data); - const page = await PageParsers.dispatch($); + const raw = await redirectHelper(url); + // pass the redirected URL to dispatcher for better error logging + const page = await PageParsers.dispatch(raw._data, raw.url); return { ...page, link: url }; }) as Promise<{ title: string; @@ -581,5 +641,5 @@ const finishArticleItem = async (item, setMpNameAsAuthor = false, skipLink = fal return item; }; -const exportedForTestingOnly = { ExtractMetadata, showTypeMapReverse }; -export { exportedForTestingOnly, fixArticleContent, fetchArticle, finishArticleItem, normalizeUrl }; +const exportedForTestingOnly = { toggleWerror, ExtractMetadata, showTypeMapReverse }; +export { exportedForTestingOnly, WeChatMpError, fixArticleContent, fetchArticle, finishArticleItem, normalizeUrl };