/** * Author: @Rongronggg9 * * There are at least three folders which are relevant with WeChat MP (Official Account Platform / Media Platform): * lib/routes/wechat * lib/routes/gov/npma * lib/routes/gzh360 * lib/routes/pku/nsd/gd * lib/routes/sdu/cs * lib/routes/nua/utils * lib/routes/hrbeu * lib/routes/freewechat * * If your new route is not in the above folders, please add it to the list. * * If your route needs to fetch MP articles from mp.weixin.qq.com, you SHOULD use `finishArticleItem`. * However, if your route need to determine some metadata by itself, you MAY use `fetchArticle`. * If you find more metadata on the webpage, consider modifying `fetchArticle` to include them. * NEVER fetch MP articles from mp.weixin.qq.com in your route in order to avoid cache key collision. * NO NEED TO use cache if you are using `finishArticleItem` or `fetchArticle`, they will handle cache for you. * * If your route fetches MP articles from other websites, you SHOULD use `fixArticleContent` to fix the content format. * If you find more fixes that should be applied, consider modifying `fixArticleContent` to include them. * * For more details of these functions, please refer to the jsDoc in the source code. */ import type { Cheerio, CheerioAPI } from 'cheerio'; import { load } from 'cheerio'; import type { Element } from 'domhandler'; import cache from '@/utils/cache'; import logger from '@/utils/logger'; import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; class WeChatMpError extends Error { constructor(message: string) { super(message); this.name = 'WeChatMpError'; } } const MAINTAINERS = ['@Rongronggg9']; const formatLogNoMention = (...params: string[]): string => `wechat-mp: ${params.join(': ')}`; const formatLog = (...params: string[]): string => `${formatLogNoMention(...params)} Consider raise an issue (mentioning ${MAINTAINERS.join(', ')}) with the article URL for further investigation`; let warn = (...params: string[]) => logger.warn(formatLog(...params)); const error = (...params: string[]): never => { const msg = formatLog(...params); logger.error(msg); throw new WeChatMpError(msg); }; const errorNoMention = (...params: string[]): never => { const msg = formatLogNoMention(...params); logger.error(msg); throw new WeChatMpError(msg); }; const toggleWerror = (() => { const onFunc = (...params: string[]) => error('WarningAsError', ...params); const offFunc = warn; return (on: boolean) => { warn = on ? onFunc : offFunc; }; })(); const replaceReturnNewline = (() => { const returnRegExp = /\r|\\(r|x0d)/g; const newlineRegExp = /\n|\\(n|x0a)/g; return (text: string, replaceReturnWith = '', replaceNewlineWith = '
') => text.replaceAll(returnRegExp, replaceReturnWith).replaceAll(newlineRegExp, replaceNewlineWith); })(); const fixUrl = (() => { const ampRegExp = /(&|\\x26)amp;/g; return (text: string) => text.replaceAll(ampRegExp, '&'); })(); class LoopContinue extends Error { constructor() { super(''); this.name = 'LoopContinue'; } } class LoopReturn extends Error { to_return: any; constructor(to_return: any) { super(''); this.name = 'LoopReturn'; this.to_return = to_return; } } const forEachScript = ($: CheerioAPI | string, callback: (script) => void, defaultReturn: any = null, selector = 'script[nonce][type="text/javascript"]') => { const scripts = typeof $ === 'string' ? [$] : $(selector).toArray(); for (const script of scripts) { try { callback(script); } catch (error) { if (error instanceof LoopReturn) { return error.to_return; } else if (error instanceof LoopContinue) { continue; } throw error; } } return defaultReturn; }; // view-source a *_SHARE_PAGE type article and search for `ITEM_SHOW_TYPE_MAP` // Please update the comments below if you find new types or new examples const showTypeMap = { // "Article". // May be combined with media, but type won't change // Combined with audio and iframe: https://mp.weixin.qq.com/s/FnjcMXZ1xdS-d6n-pUUyyw APP_MSG_PAGE: '0', // https://mp.weixin.qq.com/s?__biz=Mzg4NTA1MTkwNA==&mid=2247532942&idx=1&sn=a84e4adbe49fdb39e4d4c1b5c12a4c3f VIDEO_SHARE_PAGE: '5', MUSIC_SHARE_PAGE: '6', // https://mp.weixin.qq.com/s/FY6yQC_e4NMAxK0FBr6jwQ AUDIO_SHARE_PAGE: '7', // https://mp.weixin.qq.com/s/4p5YmYuASiQSYFiy7KqydQ // https://mp.weixin.qq.com/s?__biz=Mzg4NTA1MTkwNA==&mid=2247532936&idx=4&sn=624054c20ded6ee85c6632f419c6f758 IMG_SHARE_PAGE: '8', TEXT_SHARE_PAGE: '10', SHORT_CONTENT_PAGE: '17', }; const showTypeMapReverse = Object.fromEntries(Object.entries(showTypeMap).map(([k, v]) => [v, k])); class ExtractMetadata { private static genAssignmentRegExp = (varName: string, valuePattern: string, assignPattern: string) => new RegExp(String.raw`\b${varName}\s*${assignPattern}\s*(?["'])(?${valuePattern})\k`, 'mg'); private static genExtractFunc = ( varName: string, { valuePattern = String.raw`\w+`, assignPattern = '=', allowNotFound = false, multiple = false, }: { valuePattern?: string; assignPattern?: string; allowNotFound?: boolean; multiple?: boolean; } ) => { const regExp = this.genAssignmentRegExp(varName, valuePattern, assignPattern); return (str: string) => { const values: string[] = []; for (const match of str.matchAll(regExp)) { const value = match.groups?.value as string; if (!multiple) { return value; } values.push(value); } if (!allowNotFound && values.length === 0) { throw new LoopContinue(); } return multiple ? values : null; }; }; private static doExtract = (metadataToBeExtracted: Record string | string[] | null | undefined>, scriptText: string) => { const metadataExtracted: Record = {}; for (const [key, extractFunc] of Object.entries(metadataToBeExtracted)) { metadataExtracted[key] = extractFunc(scriptText) as string; } metadataExtracted._extractedFrom = scriptText; return metadataExtracted; }; private static commonMetadataToBeExtracted = { showType: this.genExtractFunc('item_show_type', { valuePattern: String.raw`\d+` }), realShowType: this.genExtractFunc('real_item_show_type', { valuePattern: String.raw`\d+` }), createTime: this.genExtractFunc('ct', { valuePattern: String.raw`\d+`, allowNotFound: true }), sourceUrl: this.genExtractFunc('msg_source_url', { valuePattern: `https?://[^'"]*`, allowNotFound: true }), }; static common = ($: CheerioAPI) => forEachScript( $, (script) => { const scriptText = $(script).text(); const metadataExtracted = this.doExtract(this.commonMetadataToBeExtracted, scriptText) as Record; const showType = showTypeMapReverse[metadataExtracted.showType]; const realShowType = showTypeMapReverse[metadataExtracted.realShowType]; metadataExtracted.sourceUrl = metadataExtracted.sourceUrl && fixUrl(metadataExtracted.sourceUrl); if (showType) { metadataExtracted.showType = showType; } else { warn('showType not found', `item_show_type=${metadataExtracted.showType}`); } if (realShowType) { metadataExtracted.realShowType = realShowType; } else { warn('realShowType not found', `real_item_show_type=${metadataExtracted.realShowType}`); } if (metadataExtracted.showType !== metadataExtracted.realShowType) { // never seen this happen, waiting for examples warn('showType mismatch', `item_show_type=${metadataExtracted.showType}, real_item_show_type=${metadataExtracted.realShowType}`); } throw new LoopReturn(metadataExtracted); }, {}, 'script[nonce][type="text/javascript"]:contains("real_item_show_type")' ); private static audioMetadataToBeExtracted = { voiceId: this.genExtractFunc('voiceid', { assignPattern: ':' }), duration: this.genExtractFunc('duration', { valuePattern: String.raw`\d*`, assignPattern: ':', allowNotFound: true }), }; // never seen a audio article containing multiple audio, waiting for examples static audio = ($: CheerioAPI) => forEachScript( $, (script) => { const scriptText = $(script).text(); const metadataExtracted = this.doExtract(this.audioMetadataToBeExtracted, scriptText) as Record; throw new LoopReturn(metadataExtracted); }, {}, 'script[nonce][type="text/javascript"]:contains("voiceid")' ); private static imgMetadataToBeExtracted = { imgUrls: this.genExtractFunc('cdn_url', { valuePattern: `https?://[^'"]*`, assignPattern: ':', multiple: true }), }; static img = ($: CheerioAPI) => forEachScript( $, (script) => { const scriptText = $(script).text(); const metadataExtracted = this.doExtract(this.imgMetadataToBeExtracted, scriptText) as Record; if (Array.isArray(metadataExtracted.imgUrls)) { metadataExtracted.imgUrls = metadataExtracted.imgUrls.map((url) => fixUrl(url)); } throw new LoopReturn(metadataExtracted); }, {}, 'script[nonce][type="text/javascript"]:contains("picture_page_info_list")' ); private static locationMetadataToBeExtracted = { countryName: this.genExtractFunc('countryName', { valuePattern: `[^'"]*`, assignPattern: ':' }), provinceName: this.genExtractFunc('provinceName', { valuePattern: `[^'"]*`, assignPattern: ':' }), cityName: this.genExtractFunc('cityName', { valuePattern: `[^'"]*`, assignPattern: ':' }), }; static location = ($: CheerioAPI) => forEachScript( $, (script) => { const scriptText = $(script).text(); const metadataExtracted = this.doExtract(this.locationMetadataToBeExtracted, scriptText); throw new LoopReturn(metadataExtracted); }, {}, 'script[nonce][type="text/javascript"]:contains("countryName")' ); } const replaceTag = ($, oldTag, newTagName) => { oldTag = $(oldTag); const NewTag = $($(`<${newTagName} />`)); const oldTagAttr = oldTag.attr(); for (const key in oldTagAttr) { NewTag.attr(key, oldTagAttr[key]); } NewTag.append(oldTag.contents()); oldTag.replaceWith(NewTag); }; const detectOriginalArticleUrl = ($) => { // No article content get, try the original url // example: https://mp.weixin.qq.com/s/f6sKObaZZhADTYU2Jl5Bnw if (!$('#js_content').text()) { return $('#js_share_source').attr('data-url'); } // Article content is too short, try the first link // example: https://mp.weixin.qq.com/s/9saVB4KaolRyJfpajzeFRg if ($('#js_content').text().length < 80) { return $('#js_content a').attr('href'); } return null; }; const genAudioSrc = (voiceId: string) => `https://res.wx.qq.com/voice/getvoice?mediaid=${voiceId}`; const genAudioTag = (src: string, title: string) => `