["'])(?${valuePattern})\k `, 'mg'); private static genExtractFunc = ( varName: string, { valuePattern = String.raw`\w+`, assignPattern = '=', allowNotFound = false, multiple = false, }: { valuePattern?: string; assignPattern?: string; allowNotFound?: boolean; multiple?: boolean; } ) => { const regExp = this.genAssignmentRegExp(varName, valuePattern, assignPattern); return (str: string) => { const values: string[] = []; for (const match of str.matchAll(regExp)) { const value = match.groups?.value as string; if (!multiple) { return value; } values.push(value); } if (!allowNotFound && values.length === 0) { throw new LoopContinue(); } return multiple ? values : null; }; }; private static doExtract = (metadataToBeExtracted: Recordstring | string[] | null | undefined>, scriptText: string) => { const metadataExtracted: Record = {}; for (const [key, extractFunc] of Object.entries(metadataToBeExtracted)) { metadataExtracted[key] = extractFunc(scriptText) as string; } metadataExtracted._extractedFrom = scriptText; return metadataExtracted; }; private static commonMetadataToBeExtracted = { showType: this.genExtractFunc('item_show_type', { valuePattern: String.raw`\d+` }), realShowType: this.genExtractFunc('real_item_show_type', { valuePattern: String.raw`\d+` }), createTime: this.genExtractFunc('ct', { valuePattern: String.raw`\d+`, allowNotFound: true }), sourceUrl: this.genExtractFunc('msg_source_url', { valuePattern: `https?://[^'"]*`, allowNotFound: true }), }; static common = ($: CheerioAPI) => forEachScript( $, (script) => { const scriptText = $(script).text(); const metadataExtracted = this.doExtract(this.commonMetadataToBeExtracted, scriptText) as Record ; const showType = showTypeMapReverse[metadataExtracted.showType]; const realShowType = showTypeMapReverse[metadataExtracted.realShowType]; metadataExtracted.sourceUrl = metadataExtracted.sourceUrl && fixUrl(metadataExtracted.sourceUrl); if (showType) { metadataExtracted.showType = showType; } else { warn('showType not found', `item_show_type=${metadataExtracted.showType}`); } if (realShowType) { metadataExtracted.realShowType = realShowType; } else { warn('realShowType not found', `real_item_show_type=${metadataExtracted.realShowType}`); } if (metadataExtracted.showType !== metadataExtracted.realShowType) { // never seen this happen, waiting for examples warn('showType mismatch', `item_show_type=${metadataExtracted.showType}, real_item_show_type=${metadataExtracted.realShowType}`); } throw new LoopReturn(metadataExtracted); }, {}, 'script[nonce][type="text/javascript"]:contains("real_item_show_type")' ); private static audioMetadataToBeExtracted = { voiceId: this.genExtractFunc('voiceid', { assignPattern: ':' }), duration: this.genExtractFunc('duration', { valuePattern: String.raw`\d*`, assignPattern: ':', allowNotFound: true }), }; // never seen a audio article containing multiple audio, waiting for examples static audio = ($: CheerioAPI) => forEachScript( $, (script) => { const scriptText = $(script).text(); const metadataExtracted = this.doExtract(this.audioMetadataToBeExtracted, scriptText) as Record ; throw new LoopReturn(metadataExtracted); }, {}, 'script[nonce][type="text/javascript"]:contains("voiceid")' ); private static imgMetadataToBeExtracted = { imgUrls: this.genExtractFunc('cdn_url', { valuePattern: `https?://[^'"]*`, assignPattern: ':', multiple: true }), }; static img = ($: CheerioAPI) => forEachScript( $, (script) => { const scriptText = $(script).text(); const metadataExtracted = this.doExtract(this.imgMetadataToBeExtracted, scriptText) as Record ; if (Array.isArray(metadataExtracted.imgUrls)) { metadataExtracted.imgUrls = metadataExtracted.imgUrls.map((url) => fixUrl(url)); } throw new LoopReturn(metadataExtracted); }, {}, 'script[nonce][type="text/javascript"]:contains("picture_page_info_list")' ); private static locationMetadataToBeExtracted = { countryName: this.genExtractFunc('countryName', { valuePattern: `[^'"]*`, assignPattern: ':' }), provinceName: this.genExtractFunc('provinceName', { valuePattern: `[^'"]*`, assignPattern: ':' }), cityName: this.genExtractFunc('cityName', { valuePattern: `[^'"]*`, assignPattern: ':' }), }; static location = ($: CheerioAPI) => forEachScript( $, (script) => { const scriptText = $(script).text(); const metadataExtracted = this.doExtract(this.locationMetadataToBeExtracted, scriptText); throw new LoopReturn(metadataExtracted); }, {}, 'script[nonce][type="text/javascript"]:contains("countryName")' ); } const replaceTag = ($, oldTag, newTagName) => { oldTag = $(oldTag); const NewTag = $($(`<${newTagName} />`)); const oldTagAttr = oldTag.attr(); for (const key in oldTagAttr) { NewTag.attr(key, oldTagAttr[key]); } NewTag.append(oldTag.contents()); oldTag.replaceWith(NewTag); }; const detectOriginalArticleUrl = ($) => { // No article content get, try the original url // example: https://mp.weixin.qq.com/s/f6sKObaZZhADTYU2Jl5Bnw if (!$('#js_content').text()) { return $('#js_share_source').attr('data-url'); } // Article content is too short, try the first link // example: https://mp.weixin.qq.com/s/9saVB4KaolRyJfpajzeFRg if ($('#js_content').text().length < 80) { return $('#js_content a').attr('href'); } return null; }; const genAudioSrc = (voiceId: string) => `https://res.wx.qq.com/voice/getvoice?mediaid=${voiceId}`; const genAudioTag = (src: string, title: string) => ``; const genVideoSrc = (videoId: string) => { const newSearchParams = new URLSearchParams({ origin: 'https://mp.weixin.qq.com', containerId: 'js_tx_video_container_0.3863487104715233', vid: videoId, width: '677', height: '380.8125', autoplay: 'false', allowFullScreen: 'true', chid: '17', full: 'true', show1080p: 'false', isDebugIframe: 'false', }); return `https://v.qq.com/txp/iframe/player.html?${newSearchParams.toString()}`; }; /** * Articles from WeChat MP have weird formats, this function is used to fix them. * * Even though your content are not directly fetched from WeChat MP, you SHOULD still call this function. * Calling this function is safe in most situations. * * Example usage: item.description = fixArticleContent($('div#js_content.rich_media_content')); * @param {*} html - The html to be fixed, a string or a cheerio object. * @param {boolean} skipImg - Whether to skip fixing images. * @return {string} - The fixed html, a string. */ const fixArticleContent = (html?: string | Cheerio , skipImg = false) => { let htmlResult = ''; if (typeof html === 'string') { htmlResult = html; } else if (html?.html) { htmlResult = html.html() || ''; } if (!htmlResult) { return ''; } const $ = load(htmlResult, undefined, false); if (!skipImg) { // fix img lazy loading $('img[data-src]').each((_, img) => { const $img = $(img); const realSrc = $img.attr('data-src'); if (realSrc) { $img.attr('src', realSrc); $img.removeAttr('data-src'); } }); } // fix audio: https://mp.weixin.qq.com/s/FnjcMXZ1xdS-d6n-pUUyyw $('mpvoice[voice_encode_fileid]').each((_, voice) => { const $voice = $(voice); const voiceId = $voice.attr('voice_encode_fileid'); if (voiceId) { const title = $voice.attr('name') || 'Audio'; $voice.replaceWith(genAudioTag(genAudioSrc(voiceId), title)); } }); // fix iframe: https://mp.weixin.qq.com/s/FnjcMXZ1xdS-d6n-pUUyyw $('iframe.video_iframe[data-src]').each((_, iframe) => { const $iframe = $(iframe); const dataSrc = $iframe.attr('data-src') as string; const srcUrlObj = new URL(dataSrc); if (srcUrlObj.host === 'v.qq.com' && srcUrlObj.searchParams.has('vid')) { const newSrc = genVideoSrc(srcUrlObj.searchParams.get('vid') as string); $iframe.attr('src', newSrc); $iframe.removeAttr('data-src'); const width = $iframe.attr('data-w'); const ratio = $iframe.attr('data-ratio'); if (width && ratio) { const width_ = Math.min(Number.parseInt(width), 677); $iframe.attr('width', width_.toString()); $iframe.attr('height', (width_ / Number.parseFloat(ratio)).toString()); } } // else {} FIXME: https://mp.weixin.qq.com/s?__biz=Mzg5Mjk3MzE4OQ==&mid=2247549515&idx=2&sn=a608fca597f0589c1aebd6d0b82ff6e9 }); // fix section $('section').each((_, section) => { const $section = $(section); const p_count = $section.find('p').length; const div_count = $section.find('div').length; const section_count = $section.find('section').length; if (p_count + div_count + section_count === 0) { // make it a p replaceTag($, section, 'p'); } else { // make it a div replaceTag($, section, 'div'); } }); // add breaks in code section $('code').each((_, code) => { $('
').insertAfter(code); }); // clear line index tags in code section $('.code-snippet__line-index').remove(); // clean scripts $('script').remove(); return $.html(); }; // Ref: // https://soaked.in/2020/08/wechat-platform-url/ // Known params (permanent long link): // __biz (essential), mid (essential), idx (essential), sn (essential), chksm, mpshare, scene, ascene, subscene, srcid, // lang, sharer_sharetime, sharer_shareid, version, exportkey, pass_ticket, clicktime, enterid, devicetype, nettype, // abtest_cookie, wx_header // Known params (temporary link): // src, timestamp, ver, signature, new (unessential) const normalizeUrl = (url: string, bypassHostCheck = false) => { const oriUrl = url; // already seen some weird urls with `&` escaped as `&`, so fix it // calling fixUrl should always be safe since having `&` or `\x26` in a URL is meaningless url = fixUrl(url); const urlObj = new URL(url); if (!bypassHostCheck && urlObj.host !== 'mp.weixin.qq.com') { error('URL host must be "mp.weixin.qq.com"', url); } urlObj.protocol = 'https:'; urlObj.hash = ''; // remove hash if (urlObj.pathname.startsWith('/s/')) { // a short link, just remove all the params urlObj.search = ''; } else if (urlObj.pathname === '/s') { const biz = urlObj.searchParams.get('__biz'); const mid = urlObj.searchParams.get('mid') || urlObj.searchParams.get('appmsgid'); const idx = urlObj.searchParams.get('idx') || urlObj.searchParams.get('itemidx'); const sn = urlObj.searchParams.get('sn') || urlObj.searchParams.get('sign'); if (biz && mid && idx && sn) { // a permanent long link, remove all unessential params // no need to escape anything so no need to use `new URLSearchParams({...}).toString()` urlObj.search = `?__biz=${biz}&mid=${mid}&idx=${idx}&sn=${sn}`; } else { const src = urlObj.searchParams.get('src'); const timestamp = urlObj.searchParams.get('timestamp'); const ver = urlObj.searchParams.get('ver'); const signature = urlObj.searchParams.get('signature'); if (src && timestamp && ver && signature) { // a temporary link, remove all unessential params urlObj.search = `?src=${src}×tamp=${timestamp}&ver=${ver}&signature=${signature}`; } else { warn('unknown URL search parameters', oriUrl); } } } else { warn('unknown URL path', oriUrl); } return urlObj.href; }; class PageParsers { private static common = ($: CheerioAPI, commonMetadata: Record) => { const title = replaceReturnNewline($('meta[property="og:title"]').attr('content') || '', '', ' '); const author = replaceReturnNewline($('meta[name=author]').attr('content') || '', '', ' '); const pubDate = commonMetadata.createTime ? parseDate(Number.parseInt(commonMetadata.createTime) * 1000) : undefined; const mpName = $('.wx_follow_nickname').first().text()?.trim(); let summary = replaceReturnNewline($('meta[name=description]').attr('content') || ''); const description = summary; summary = summary.replaceAll('
', ' ') === title ? '' : summary; return { title, author, description, summary, pubDate, mpName } as { title: string; author: string; description: string; summary: string; pubDate?: Date; mpName?: string; enclosure_url?: string; itunes_duration?: string | number; enclosure_type?: string; }; }; private static appMsg = async ($: CheerioAPI, commonMetadata: Record) => { const page = PageParsers.common($, commonMetadata); page.description = fixArticleContent($('#js_content')); const originalArticleUrl = detectOriginalArticleUrl($); if (originalArticleUrl) { // No article or article is too short, try to fetch the description from the original article const data = await ofetch(normalizeUrl(originalArticleUrl)); const original$ = load(data); page.description += fixArticleContent(original$('#js_content')); } return page; }; private static img = ($: CheerioAPI, commonMetadata: Record ) => { const page = PageParsers.common($, commonMetadata); const imgUrls = ExtractMetadata.img($)?.imgUrls; let imgHtml = ''; if (Array.isArray(imgUrls) && imgUrls.length > 0) { for (const imgUrl of imgUrls) { imgHtml += ` `; } } page.description += imgHtml; return page; }; private static audio = ($: CheerioAPI, commonMetadata: Record
) => { const page = PageParsers.common($, commonMetadata); const audioMetadata = ExtractMetadata.audio($); const audioUrl = genAudioSrc(audioMetadata.voiceId); page.enclosure_url = audioUrl; page.itunes_duration = audioMetadata.duration; page.enclosure_type = 'audio/mp3'; // FIXME: may it be other types? page.description += '
' + genAudioTag(audioUrl, page.title); return page; }; private static fallback = ($: CheerioAPI, commonMetadata: Record) => { const page = PageParsers.common($, commonMetadata); const image = $('meta[property="og:image"]').attr('content'); if (image) { page.description += ` `; } return page; }; static dispatch = async (html: string, url: string) => { const $ = load(html); const commonMetadata = ExtractMetadata.common($); let page: Record
; let pageText: string, pageTextShort: string; switch (commonMetadata.showType) { case 'APP_MSG_PAGE': page = await PageParsers.appMsg($, commonMetadata); break; case 'AUDIO_SHARE_PAGE': page = PageParsers.audio($, commonMetadata); break; case 'IMG_SHARE_PAGE': page = PageParsers.img($, commonMetadata); break; case 'VIDEO_SHARE_PAGE': page = PageParsers.fallback($, commonMetadata); break; case undefined: $('script, style').remove(); pageText = $('title, body').text().replaceAll(/\s+/g, ' ').trim(); pageTextShort = pageText.slice(0, 25); if (pageText.length >= 25 + '...'.length) { pageTextShort = pageText.slice(0, 25); pageTextShort += '...'; } if (pageText.includes('已被发布者删除')) { errorNoMention('deleted by author', pageTextShort, url); } else if (new URL(url).pathname.includes('captcha') || pageText.includes('环境异常')) { errorNoMention('request blocked by WAF', pageTextShort, url); } else { error('unknown page, probably due to WAF', pageTextShort, url); } /* v8 ignore next */ return {}; // just to make TypeScript happy, actually UNREACHABLE default: warn('new showType, trying fallback method', `showType=${commonMetadata.showType}`, url); page = PageParsers.fallback($, commonMetadata); } const locationMetadata = ExtractMetadata.location($); let location = ''; for (const loc of [locationMetadata.countryName, locationMetadata.provinceName, locationMetadata.cityName]) { if (loc) { location += loc + ' '; } } location = location.trim(); if (location) { page.description += ` 📍发表于:${location}
`; } if (commonMetadata.sourceUrl) { page.description += ``; } return page; }; } const redirectHelper = async (url: string, maxRedirects: number = 5) => { maxRedirects--; const raw = await ofetch.raw(url); if ([301, 302, 303, 307, 308].includes(raw.status)) { if (!raw.headers.has('location')) { error('redirect without location', url); } else if (maxRedirects <= 0) { error('too many redirects', url); } return await redirectHelper(raw.headers.get('location') as string, maxRedirects); } return raw; }; /** * Fetch article and its metadata from WeChat MP (mp.weixin.qq.com). * * If you use this function, no need to call `fixArticleContent` * @param url - The url of the article. * @param bypassHostCheck - Whether to bypass host check. * @return - An object containing the article and its metadata. */ const fetchArticle = (url: string, bypassHostCheck: boolean = false) => { url = normalizeUrl(url, bypassHostCheck); return cache.tryGet(url, async () => { const raw = await redirectHelper(url); // pass the redirected URL to dispatcher for better error logging const page = await PageParsers.dispatch(raw._data, raw.url); return { ...page, link: url }; }) as Promise<{ title: string; author: string; description: string; summary: string; pubDate?: Date; mpName?: string; link: string; enclosure_type?: string; enclosure_url?: string; itunes_duration?: string | number; }>; }; /** * Fetch article and its metadata from WeChat MP (mp.weixin.qq.com), then fill the `item` object with the result. * * If you use this function, no need to call `fetchArticle` or `fixArticleContent` * * A new route SHOULD use this function instead of manually calling the above functions * * An existing route adopting this function SHOULD either: * - set `skipLink` to true (not recommended) * - set `item.guid` to `item.link` BEFORE calling this function * @param {object} ctx - The context object. * @param {object} item - The item object to be filled. * @param {boolean} setMpNameAsAuthor - If `true`, `author` will be the MP itself, otherwise the real author of the article. * @param {boolean} skipLink - Whether to skip overriding `item.link` with the normalized url. * @return {Promise