xiaoxue_admin
/
RSSHub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

test: wechat-mp

This commit is contained in:
DIYgod 2024-02-25 02:08:35 +08:00
parent bdf8ace129
commit 01e142ccaf
No known key found for this signature in database
22 changed files with 60 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
test/utils/wechat-mp.js → lib/utils/wechat-mp.test.ts
View File

@ -1,17 +1,7 @@
process.env.REQUEST_TIMEOUT = '500';
const cheerio = require('cheerio');
const {
_internal: { normalizeUrl },
fetchArticle,
finishArticleItem,
fixArticleContent,
} = require('../../lib/utils/wechat-mp');
const nock = require('nock');
const ctx = require('../../lib/app').context;
afterAll(() => {
delete process.env.REQUEST_TIMEOUT;
});
import { describe, expect, it } from '@jest/globals';
import { load } from 'cheerio';
import nock from 'nock';
import { fixArticleContent, fetchArticle, finishArticleItem, normalizeUrl } from '@/utils/wechat-mp';
// date from the cache will be an ISO8601 string, so we need to use this function
const compareDate = (date1, date2) => {
@ -56,20 +46,19 @@ describe('wechat-mp', () => {
'<script>const test = "test"</script>';
const expectedHtmlSection =
expectedCodeSection + '<p>test</p>' + '<div><p>test</p></div>' + '<div><div>test</div></div>' + '<div><div><p>test</p></div></div>' + '<div><div><p>test</p></div></div>' + '<p>test</p>' + '<div><p>test</p></div>';
let $ = cheerio.load(divHeader + htmlSection + divFooter);
let $ = load(divHeader + htmlSection + divFooter);
expect(fixArticleContent(htmlSection)).toBe(expectedHtmlSection);
expect(fixArticleContent($('div#js_content.rich_media_content'))).toBe(expectedHtmlSection);
const htmlImg = '<img alt="test" data-src="http://rsshub.test/test.jpg" src="http://rsshub.test/test.jpg">' + '<img alt="test" data-src="http://rsshub.test/test.jpg">' + '<img alt="test" src="http://rsshub.test/test.jpg">';
const expectedHtmlImg = Array.from({ length: 3 + 1 }).join('<img alt="test" src="http://rsshub.test/test.jpg">');
$ = cheerio.load(divHeader + htmlImg + divFooter);
$ = load(divHeader + htmlImg + divFooter);
expect(fixArticleContent(htmlImg)).toBe(expectedHtmlImg);
expect(fixArticleContent($('div#js_content.rich_media_content'))).toBe(expectedHtmlImg);
expect(fixArticleContent(htmlImg, true)).toBe(htmlImg);
expect(fixArticleContent($('div#js_content.rich_media_content'), true)).toBe(htmlImg);
expect(fixArticleContent('')).toBe('');
expect(fixArticleContent(null)).toBe('');
expect(fixArticleContent()).toBe('');
expect(fixArticleContent($('div#something_not_in.the_document_tree'))).toBe('');
});
@ -121,7 +110,14 @@ describe('wechat-mp', () => {
const httpsUrl = 'https://mp.weixin.qq.com/rsshub_test/wechatMp_fetchArticle';
const httpUrl = httpsUrl.replace(/^https:\/\//, 'http://');
const expectedItem = {
const expectedItem: {
title: string;
summary: string;
author: string;
description: string;
mpName?: string;
link: string;
} = {
title: 'title',
summary: 'summary',
author: 'author',
@ -131,13 +127,13 @@ describe('wechat-mp', () => {
};
const expectedDate = new Date(ct * 1000);
const fetchArticleItem = await fetchArticle(ctx, httpUrl);
const fetchArticleItem = await fetchArticle(httpUrl);
expect(compareDate(fetchArticleItem.pubDate, expectedDate)).toBe(true);
delete fetchArticleItem.pubDate;
expect(fetchArticleItem).toEqual(expectedItem);
delete expectedItem.mpName;
const finishedArticleItem = await finishArticleItem(ctx, { link: httpUrl });
const finishedArticleItem = await finishArticleItem({ link: httpUrl });
expect(compareDate(finishedArticleItem.pubDate, expectedDate)).toBe(true);
delete finishedArticleItem.pubDate;
expect(finishedArticleItem).toEqual(expectedItem);

34
lib/utils/wechat-mp.ts
View File

@ -26,8 +26,9 @@
*/
import got from '@/utils/got';
import { load } from 'cheerio';
import { load, type Cheerio, type Element } from 'cheerio';
import { parseDate } from '@/utils/parse-date';
import cache from '@/utils/cache';
const replaceTag = ($, oldTag, newTagName) => {
oldTag = $(oldTag);
@ -76,12 +77,17 @@ const detectSourceUrl = ($) => {
* @param {boolean} skipImg - Whether to skip fixing images.
* @return {string} - The fixed html, a string.
*/
const fixArticleContent = (html, skipImg = false) => {
html = html && html.html ? html.html() : html; // do not disturb the original tree
if (!html) {
const fixArticleContent = (html?: string | Cheerio<Element>, skipImg = false) => {
let htmlResult = '';
if (typeof html === 'string') {
htmlResult = html;
} else if (html?.html) {
htmlResult = html.html() || '';
}
if (!htmlResult) {
return '';
}
const $ = load(html, undefined, false);
const $ = load(htmlResult, undefined, false);
if (!skipImg) {
// fix img lazy loading
$('img[data-src]').each((_, img) => {
@ -187,9 +193,9 @@ const normalizeUrl = (url, bypassHostCheck = false) => {
* @param {boolean} bypassHostCheck - Whether to bypass host check.
* @return {Promise<object>} - An object containing the article and its metadata.
*/
const fetchArticle = (ctx, url, bypassHostCheck = false) => {
const fetchArticle = (url, bypassHostCheck = false) => {
url = normalizeUrl(url, bypassHostCheck);
return ctx.cache.tryGet(url, async () => {
return cache.tryGet(url, async () => {
const response = await got(url);
// @ts-expect-error custom field
const $ = load(response.data);
@ -225,7 +231,15 @@ const fetchArticle = (ctx, url, bypassHostCheck = false) => {
let mpName = $('.profile_nickname').first().text();
mpName = mpName && mpName.trim();
return { title, author, description, summary, pubDate, mpName, link: response.url };
});
}) as Promise<{
title: string;
author: string;
description: string;
summary: string;
pubDate?: Date;
mpName?: string;
link: string;
}>;
};
/**
@ -244,8 +258,8 @@ const fetchArticle = (ctx, url, bypassHostCheck = false) => {
* @param {boolean} skipLink - Whether to skip overriding `item.link` with the normalized url.
* @return {Promise<object>} - The incoming `item` object, with the article and its metadata filled in.
*/
const finishArticleItem = async (ctx, item, setMpNameAsAuthor = false, skipLink = false) => {
const { title, author, description, summary, pubDate, mpName, link } = await fetchArticle(ctx, item.link);
const finishArticleItem = async (item, setMpNameAsAuthor = false, skipLink = false) => {
const { title, author, description, summary, pubDate, mpName, link } = await fetchArticle(item.link);
item.title = title || item.title;
item.description = description || item.description;
item.summary = summary || item.summary;

2
lib/v2/gov/general/general.js
View File

@ -187,7 +187,7 @@ const gdgov = async (info, ctx) => {
});
} else if (idlink.host === 'mp.weixin.qq.com') {
const { finishArticleItem } = require('@/utils/wechat-mp');
return finishArticleItem(ctx, { link });
return finishArticleItem({ link });
} else {
return ctx.cache.tryGet(link, async () => {
// 获取网页

2
lib/v2/gov/nmpa/generic.js
View File

@ -45,7 +45,7 @@ module.exports = async (ctx) => {
return item;
});
} else if (/^https:\/\/mp\.weixin\.qq\.com\//.test(item.link)) {
return finishArticleItem(ctx, item);
return finishArticleItem(item);
} else {
return item;
}

2
lib/v2/hrbeu/job/list.js
View File

@ -48,7 +48,7 @@ module.exports = async (ctx) => {
const content = cheerio.load(detailResponse.data);
item.description = content('.article').html();
} else if (new URL(item.link).hostname === 'mp.weixin.qq.com') {
await finishArticleItem(ctx, item);
await finishArticleItem(item);
} else {
item.description = '本文需跳转，请点击标题后阅读';
}

2
lib/v2/hrbeu/uae/news.js
View File

@ -49,7 +49,7 @@ module.exports = async (ctx) => {
const $1 = cheerio.load(resp.data);
item.description = $1('div#print').html();
} else if (new URL(item.link).hostname === 'mp.weixin.qq.com') {
await finishArticleItem(ctx, item);
await finishArticleItem(item);
} else {
item.description = '本文需跳转，请点击标题后阅读';
}

2
lib/v2/newrank/wechat.js
View File

@ -55,7 +55,7 @@ module.exports = async (ctx) => {
link: item.url,
pubDate: item.publicTime,
}));
await Promise.all(items.map((item) => finishArticleItem(ctx, item)));
await Promise.all(items.map((item) => finishArticleItem(item)));
ctx.state.data = {
title: name + ' - 微信公众号',

2
lib/v2/nua/utils.js
View File

@ -59,7 +59,7 @@ const ProcessFeed = (items, artiContent, ctx) =>
return item;
}
case 'wechat-mp':
return fetchArticle(ctx, item.link);
return fetchArticle(item.link);
case 'unknown':
default:
item.description = `暂不支持解析该内容，请点击 ${arti_link('原文', item.link)}`;

2
lib/v2/pku/nsd.js
View File

@ -40,7 +40,7 @@ module.exports = async (ctx) => {
list.map((item) => {
switch (item.type) {
case 'wechat-mp':
return finishArticleItem(ctx, item);
return finishArticleItem(item);
case 'pku-news':
return ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({ url: item.link, https: { rejectUnauthorized: false } });

2
lib/v2/sdu/cs.js
View File

@ -31,7 +31,7 @@ module.exports = async (ctx) => {
item.map((item) =>
ctx.cache.tryGet(item.link, async () => {
if (new URL(item.link).hostname === 'mp.weixin.qq.com') {
return finishArticleItem(ctx, item);
return finishArticleItem(item);
} else if (new URL(item.link).hostname !== 'www.cs.sdu.edu.cn') {
return item;
}

2
lib/v2/sjtu/gs.js
View File

@ -36,7 +36,7 @@ module.exports = async (ctx) => {
list.map((item) =>
ctx.cache.tryGet(item.link, async () => {
if (new URL(item.link).hostname === 'mp.weixin.qq.com') {
item.description = (await fetchArticle(ctx, item.link)).description;
item.description = (await fetchArticle(item.link)).description;
} else {
const detailResponse = await got({
method: 'get',

2
lib/v2/slowmist/slowmist.js
View File

@ -39,7 +39,7 @@ module.exports = async (ctx) => {
pubDate: parseDate(item.date),
}));
items = await Promise.all(items.map((item) => finishArticleItem(ctx, item)));
items = await Promise.all(items.map((item) => finishArticleItem(item)));
ctx.state.data = {
title,

2
lib/v2/tju/oaa/index.js
View File

@ -82,7 +82,7 @@ module.exports = async (ctx) => {
list.map((item) => {
switch (item.type) {
case 'wechat-mp':
return finishArticleItem(ctx, item);
return finishArticleItem(item);
case 'tju-oaa':
case 'in-site':
return ctx.cache.tryGet(item.link, async () => {

2
lib/v2/wechat/data258.js
View File

@ -119,7 +119,7 @@ module.exports = async (ctx) => {
throw err;
}
await Promise.all(items.map((item) => finishArticleItem(ctx, item, !!categoryPage)));
await Promise.all(items.map((item) => finishArticleItem(item, !!categoryPage)));
ctx.state.data = {
title,

2
lib/v2/wechat/ershcimi.js
View File

@ -24,7 +24,7 @@ module.exports = async (ctx) => {
})
.get();
await Promise.all(items.map((item) => finishArticleItem(ctx, item)));
await Promise.all(items.map((item) => finishArticleItem(item)));
ctx.state.data = {
title: `微信公众号 - ${$('span.name').text()}`,

2
lib/v2/wechat/feeddd.js
View File

@ -27,7 +27,7 @@ module.exports = async (ctx) => {
guid: item.id,
}));
items = await Promise.all(items.map((item) => finishArticleItem(ctx, item)));
items = await Promise.all(items.map((item) => finishArticleItem(item)));
ctx.state.data = {
title: response.data.title,

2
lib/v2/wechat/feeds.js
View File

@ -12,7 +12,7 @@ module.exports = async (ctx) => {
link: item.link,
guid: item.link,
}));
await Promise.all(items.map((item) => finishArticleItem(ctx, item)));
await Promise.all(items.map((item) => finishArticleItem(item)));
ctx.state.data = {
title: feed.title,

2
lib/v2/wechat/mp.js
View File

@ -32,7 +32,7 @@ module.exports = async (ctx) => {
link: item.link,
guid: item.link,
};
return finishArticleItem(ctx, single);
return finishArticleItem(single);
})
);
ctx.state.data = {

2
lib/v2/wechat/msgalbum.js
View File

@ -23,7 +23,7 @@ module.exports = async (ctx) => {
link,
guid: link,
};
return finishArticleItem(ctx, single);
return finishArticleItem(single);
})
);
ctx.state.data = {

2
lib/v2/wechat/sogou.js
View File

@ -47,7 +47,7 @@ module.exports = async (ctx) => {
guid: link,
};
await finishArticleItem(ctx, item);
await finishArticleItem(item);
ctx.state.data = {
title: `${title} 的微信公众号`,

2
lib/v2/wechat/tgchannel.js
View File

@ -126,7 +126,7 @@ module.exports = async (ctx) => {
if (link !== undefined) {
try {
return await finishArticleItem(ctx, single);
return await finishArticleItem(single);
} catch {
single.description = item.find('.tgme_widget_message_text').html();
}

2
lib/v2/wechat/wechat2rss.js
View File

@ -16,7 +16,7 @@ module.exports = async (ctx) => {
link: i.link,
}));
items = await Promise.all(items.map((item) => finishArticleItem(ctx, item)));
items = await Promise.all(items.map((item) => finishArticleItem(item)));
ctx.state.data = {
title,